diff --git a/devel/CHANGELOG/index.html b/devel/CHANGELOG/index.html
index 09a4e756e..1a711a720 100644
--- a/devel/CHANGELOG/index.html
+++ b/devel/CHANGELOG/index.html
@@ -3147,14 +3147,38 @@
     </label>
     <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
       
+        <li class="md-nav__item">
+  <a href="#unreleased" class="md-nav__link">
+    <span class="md-ellipsis">
+      Unreleased
+    </span>
+  </a>
+  
+    <nav class="md-nav" aria-label="Unreleased">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#fixed" class="md-nav__link">
+    <span class="md-ellipsis">
+      Fixed
+    </span>
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
         <li class="md-nav__item">
   <a href="#090-new-methods-better-docs-and-bugfixes" class="md-nav__link">
     <span class="md-ellipsis">
-      0.9.0 🆕 New methods, better docs and bugfixes 📚🐞
+      0.9.0 - 🆕 New methods, better docs and bugfixes 📚🐞
     </span>
   </a>
   
-    <nav class="md-nav" aria-label="0.9.0 🆕 New methods, better docs and bugfixes 📚🐞">
+    <nav class="md-nav" aria-label="0.9.0 - 🆕 New methods, better docs and bugfixes 📚🐞">
       <ul class="md-nav__list">
         
           <li class="md-nav__item">
@@ -3167,7 +3191,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed" class="md-nav__link">
+  <a href="#fixed_1" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3209,7 +3233,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed_1" class="md-nav__link">
+  <a href="#fixed_2" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3260,7 +3284,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed_2" class="md-nav__link">
+  <a href="#fixed_3" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3293,7 +3317,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed_3" class="md-nav__link">
+  <a href="#fixed_4" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3344,7 +3368,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed_4" class="md-nav__link">
+  <a href="#fixed_5" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3431,7 +3455,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed_5" class="md-nav__link">
+  <a href="#fixed_6" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3516,14 +3540,38 @@
     </label>
     <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
       
+        <li class="md-nav__item">
+  <a href="#unreleased" class="md-nav__link">
+    <span class="md-ellipsis">
+      Unreleased
+    </span>
+  </a>
+  
+    <nav class="md-nav" aria-label="Unreleased">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#fixed" class="md-nav__link">
+    <span class="md-ellipsis">
+      Fixed
+    </span>
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
         <li class="md-nav__item">
   <a href="#090-new-methods-better-docs-and-bugfixes" class="md-nav__link">
     <span class="md-ellipsis">
-      0.9.0 🆕 New methods, better docs and bugfixes 📚🐞
+      0.9.0 - 🆕 New methods, better docs and bugfixes 📚🐞
     </span>
   </a>
   
-    <nav class="md-nav" aria-label="0.9.0 🆕 New methods, better docs and bugfixes 📚🐞">
+    <nav class="md-nav" aria-label="0.9.0 - 🆕 New methods, better docs and bugfixes 📚🐞">
       <ul class="md-nav__list">
         
           <li class="md-nav__item">
@@ -3536,7 +3584,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed" class="md-nav__link">
+  <a href="#fixed_1" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3578,7 +3626,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed_1" class="md-nav__link">
+  <a href="#fixed_2" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3629,7 +3677,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed_2" class="md-nav__link">
+  <a href="#fixed_3" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3662,7 +3710,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed_3" class="md-nav__link">
+  <a href="#fixed_4" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3713,7 +3761,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed_4" class="md-nav__link">
+  <a href="#fixed_5" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3800,7 +3848,7 @@
 </li>
         
           <li class="md-nav__item">
-  <a href="#fixed_5" class="md-nav__link">
+  <a href="#fixed_6" class="md-nav__link">
     <span class="md-ellipsis">
       Fixed
     </span>
@@ -3841,7 +3889,14 @@
 
 
 <h1 id="changelog">Changelog<a class="headerlink" href="#changelog" title="Permanent link">&para;</a></h1>
-<h2 id="090-new-methods-better-docs-and-bugfixes">0.9.0 🆕 New methods, better docs and bugfixes 📚🐞<a class="headerlink" href="#090-new-methods-better-docs-and-bugfixes" title="Permanent link">&para;</a></h2>
+<h2 id="unreleased">Unreleased<a class="headerlink" href="#unreleased" title="Permanent link">&para;</a></h2>
+<h3 id="fixed">Fixed<a class="headerlink" href="#fixed" title="Permanent link">&para;</a></h3>
+<ul>
+<li><code>FutureWarning</code> for <code>ParallelConfig</code> constantly raised without actually 
+  instantiating the object
+  <a href="https://github.com/aai-institute/pyDVL/pull/562">PR #562</a></li>
+</ul>
+<h2 id="090-new-methods-better-docs-and-bugfixes">0.9.0 - 🆕 New methods, better docs and bugfixes 📚🐞<a class="headerlink" href="#090-new-methods-better-docs-and-bugfixes" title="Permanent link">&para;</a></h2>
 <h3 id="added">Added<a class="headerlink" href="#added" title="Permanent link">&para;</a></h3>
 <ul>
 <li>New method <code>MSR Banzhaf</code> with accompanying notebook, and new stopping
@@ -3859,7 +3914,7 @@ <h3 id="added">Added<a class="headerlink" href="#added" title="Permanent link">&
 <li>Documentation about writing notes for new features, changes or deprecations
   <a href="https://github.com/aai-institute/pyDVL/pull/557">PR #557</a></li>
 </ul>
-<h3 id="fixed">Fixed<a class="headerlink" href="#fixed" title="Permanent link">&para;</a></h3>
+<h3 id="fixed_1">Fixed<a class="headerlink" href="#fixed_1" title="Permanent link">&para;</a></h3>
 <ul>
 <li>Bug in <code>LissaInfluence</code>, when not using CPU device
   <a href="https://github.com/aai-institute/pyDVL/pull/495">PR #495</a></li>
@@ -3894,7 +3949,7 @@ <h3 id="added_1">Added<a class="headerlink" href="#added_1" title="Permanent lin
 <li>Implemented exact games in Castro et al. 2009 and 2017
   <a href="https://github.com/appliedAI-Initiative/pyDVL/pull/341">PR #341</a></li>
 </ul>
-<h3 id="fixed_1">Fixed<a class="headerlink" href="#fixed_1" title="Permanent link">&para;</a></h3>
+<h3 id="fixed_2">Fixed<a class="headerlink" href="#fixed_2" title="Permanent link">&para;</a></h3>
 <ul>
 <li>Bug in using <code>DaskInfluenceCalcualator</code> with <code>TorchnumpyConverter</code>
   for single dimensional arrays 
@@ -3944,7 +3999,7 @@ <h3 id="changed_2">Changed<a class="headerlink" href="#changed_2" title="Permane
 </ul>
 </li>
 </ul>
-<h3 id="fixed_2">Fixed<a class="headerlink" href="#fixed_2" title="Permanent link">&para;</a></h3>
+<h3 id="fixed_3">Fixed<a class="headerlink" href="#fixed_3" title="Permanent link">&para;</a></h3>
 <ul>
 <li>Import bug in README <a href="https://github.com/aai-institute/pyDVL/issues/457">PR #457</a></li>
 </ul>
@@ -3961,7 +4016,7 @@ <h3 id="added_3">Added<a class="headerlink" href="#added_3" title="Permanent lin
 <li>Faster semi-value computation with per-index check of stopping criteria (optional)
   <a href="https://github.com/aai-institute/pyDVL/pull/437">PR #437</a></li>
 </ul>
-<h3 id="fixed_3">Fixed<a class="headerlink" href="#fixed_3" title="Permanent link">&para;</a></h3>
+<h3 id="fixed_4">Fixed<a class="headerlink" href="#fixed_4" title="Permanent link">&para;</a></h3>
 <ul>
 <li>Fix initialization of <code>data_names</code> in <code>ValuationResult.zeros()</code>
   <a href="https://github.com/aai-institute/pyDVL/pull/443">PR #443</a></li>
@@ -4025,7 +4080,7 @@ <h3 id="changed_4">Changed<a class="headerlink" href="#changed_4" title="Permane
 <li>Bump torch dependency for influence package to 2.0
     <a href="https://github.com/aai-institute/pyDVL/pull/365">PR #365</a></li>
 </ul>
-<h3 id="fixed_4">Fixed<a class="headerlink" href="#fixed_4" title="Permanent link">&para;</a></h3>
+<h3 id="fixed_5">Fixed<a class="headerlink" href="#fixed_5" title="Permanent link">&para;</a></h3>
 <ul>
 <li>Fixes to parallel computation of generic semi-values: properly handle all
   samplers and stopping criteria, irrespective of parallel backend.
@@ -4189,7 +4244,7 @@ <h3 id="changed_5">Changed<a class="headerlink" href="#changed_5" title="Permane
   readme and documentation for contributors. Typos, grammar and style in code,
   documentation and notebooks.
 - Internal renaming and rearranging in the parallelization and caching modules.</p>
-<h3 id="fixed_5">Fixed<a class="headerlink" href="#fixed_5" title="Permanent link">&para;</a></h3>
+<h3 id="fixed_6">Fixed<a class="headerlink" href="#fixed_6" title="Permanent link">&para;</a></h3>
 <ul>
 <li>Bug in random matrix generation
   <a href="https://github.com/aai-institute/pyDVL/pull/161">PR #161</a>.</li>
@@ -4240,7 +4295,7 @@ <h2 id="010-first-release">0.1.0 - 🎉 first release<a class="headerlink" href=
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/index.html b/devel/api/pydvl/index.html
index b9e5a91ee..a4eb279c7 100644
--- a/devel/api/pydvl/index.html
+++ b/devel/api/pydvl/index.html
@@ -3258,7 +3258,7 @@ <h2 id="pydvl--the-python-data-valuation-library-api">The Python Data Valuation
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3268,7 +3268,7 @@ <h2 id="pydvl--the-python-data-valuation-library-api">The Python Data Valuation
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/influence/array/index.html b/devel/api/pydvl/influence/array/index.html
index bcecfc3c9..715754715 100644
--- a/devel/api/pydvl/influence/array/index.html
+++ b/devel/api/pydvl/influence/array/index.html
@@ -5142,7 +5142,7 @@ <h3 id="pydvl.influence.array.NestedLazyChunkSequence.to_zarr" class="doc doc-he
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -5152,7 +5152,7 @@ <h3 id="pydvl.influence.array.NestedLazyChunkSequence.to_zarr" class="doc doc-he
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/influence/base_influence_function_model/index.html b/devel/api/pydvl/influence/base_influence_function_model/index.html
index 5c4824d1a..50544996d 100644
--- a/devel/api/pydvl/influence/base_influence_function_model/index.html
+++ b/devel/api/pydvl/influence/base_influence_function_model/index.html
@@ -3934,7 +3934,7 @@ <h3 id="pydvl.influence.base_influence_function_model.InfluenceFunctionModel.inf
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3944,7 +3944,7 @@ <h3 id="pydvl.influence.base_influence_function_model.InfluenceFunctionModel.inf
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/influence/index.html b/devel/api/pydvl/influence/index.html
index 00fe93b61..ef6e8696d 100644
--- a/devel/api/pydvl/influence/index.html
+++ b/devel/api/pydvl/influence/index.html
@@ -3247,7 +3247,7 @@ <h1 id="pydvl.influence" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3257,7 +3257,7 @@ <h1 id="pydvl.influence" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/influence/influence_calculator/index.html b/devel/api/pydvl/influence/influence_calculator/index.html
index 79e006c76..6e52aac12 100644
--- a/devel/api/pydvl/influence/influence_calculator/index.html
+++ b/devel/api/pydvl/influence/influence_calculator/index.html
@@ -5415,7 +5415,7 @@ <h3 id="pydvl.influence.influence_calculator.SequentialInfluenceCalculator.influ
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -5425,7 +5425,7 @@ <h3 id="pydvl.influence.influence_calculator.SequentialInfluenceCalculator.influ
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/influence/torch/functional/index.html b/devel/api/pydvl/influence/torch/functional/index.html
index a5b3075b9..193672e5e 100644
--- a/devel/api/pydvl/influence/torch/functional/index.html
+++ b/devel/api/pydvl/influence/torch/functional/index.html
@@ -7579,7 +7579,7 @@ <h2 id="pydvl.influence.torch.functional.model_hessian_nystroem_approximation" c
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -7589,7 +7589,7 @@ <h2 id="pydvl.influence.torch.functional.model_hessian_nystroem_approximation" c
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/influence/torch/index.html b/devel/api/pydvl/influence/torch/index.html
index 60a676377..5fbd1e415 100644
--- a/devel/api/pydvl/influence/torch/index.html
+++ b/devel/api/pydvl/influence/torch/index.html
@@ -3239,7 +3239,7 @@ <h1 id="pydvl.influence.torch" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3249,7 +3249,7 @@ <h1 id="pydvl.influence.torch" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/influence/torch/influence_function_model/index.html b/devel/api/pydvl/influence/torch/influence_function_model/index.html
index e5fc29e2f..cd69105e8 100644
--- a/devel/api/pydvl/influence/torch/influence_function_model/index.html
+++ b/devel/api/pydvl/influence/torch/influence_function_model/index.html
@@ -11302,7 +11302,7 @@ <h3 id="pydvl.influence.torch.influence_function_model.NystroemSketchInfluence.i
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -11312,7 +11312,7 @@ <h3 id="pydvl.influence.torch.influence_function_model.NystroemSketchInfluence.i
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/influence/torch/pre_conditioner/index.html b/devel/api/pydvl/influence/torch/pre_conditioner/index.html
index 54ad18543..8ff18ccf4 100644
--- a/devel/api/pydvl/influence/torch/pre_conditioner/index.html
+++ b/devel/api/pydvl/influence/torch/pre_conditioner/index.html
@@ -4321,7 +4321,7 @@ <h3 id="pydvl.influence.torch.pre_conditioner.NystroemPreConditioner.fit" class=
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -4331,7 +4331,7 @@ <h3 id="pydvl.influence.torch.pre_conditioner.NystroemPreConditioner.fit" class=
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/influence/torch/util/index.html b/devel/api/pydvl/influence/torch/util/index.html
index 409e514a6..23d77e8de 100644
--- a/devel/api/pydvl/influence/torch/util/index.html
+++ b/devel/api/pydvl/influence/torch/util/index.html
@@ -5784,7 +5784,7 @@ <h2 id="pydvl.influence.torch.util.empirical_cross_entropy_loss_fn" class="doc d
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -5794,7 +5794,7 @@ <h2 id="pydvl.influence.torch.util.empirical_cross_entropy_loss_fn" class="doc d
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/parallel/backend/index.html b/devel/api/pydvl/parallel/backend/index.html
index 92a453c5a..f7e02cde3 100644
--- a/devel/api/pydvl/parallel/backend/index.html
+++ b/devel/api/pydvl/parallel/backend/index.html
@@ -3870,7 +3870,7 @@ <h2 id="pydvl.parallel.backend.available_cpus" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3880,7 +3880,7 @@ <h2 id="pydvl.parallel.backend.available_cpus" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/parallel/backends/index.html b/devel/api/pydvl/parallel/backends/index.html
index dbb813e36..e7b88b7aa 100644
--- a/devel/api/pydvl/parallel/backends/index.html
+++ b/devel/api/pydvl/parallel/backends/index.html
@@ -3239,7 +3239,7 @@ <h1 id="pydvl.parallel.backends" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3249,7 +3249,7 @@ <h1 id="pydvl.parallel.backends" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/parallel/backends/joblib/index.html b/devel/api/pydvl/parallel/backends/joblib/index.html
index c1587d01a..e3d8f71fc 100644
--- a/devel/api/pydvl/parallel/backends/joblib/index.html
+++ b/devel/api/pydvl/parallel/backends/joblib/index.html
@@ -3742,7 +3742,7 @@ <h3 id="pydvl.parallel.backends.joblib.JoblibParallelBackend.wrap" class="doc do
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3752,7 +3752,7 @@ <h3 id="pydvl.parallel.backends.joblib.JoblibParallelBackend.wrap" class="doc do
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/parallel/backends/ray/index.html b/devel/api/pydvl/parallel/backends/ray/index.html
index 739a6e486..41636ee08 100644
--- a/devel/api/pydvl/parallel/backends/ray/index.html
+++ b/devel/api/pydvl/parallel/backends/ray/index.html
@@ -3772,7 +3772,7 @@ <h3 id="pydvl.parallel.backends.ray.RayParallelBackend.wrap" class="doc doc-head
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3782,7 +3782,7 @@ <h3 id="pydvl.parallel.backends.ray.RayParallelBackend.wrap" class="doc doc-head
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/parallel/config/index.html b/devel/api/pydvl/parallel/config/index.html
index 13e93d5f9..7de1e16a3 100644
--- a/devel/api/pydvl/parallel/config/index.html
+++ b/devel/api/pydvl/parallel/config/index.html
@@ -3456,7 +3456,7 @@ <h2 id="pydvl.parallel.config.ParallelConfig" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3466,7 +3466,7 @@ <h2 id="pydvl.parallel.config.ParallelConfig" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/parallel/futures/index.html b/devel/api/pydvl/parallel/futures/index.html
index 5ecfc6499..054275314 100644
--- a/devel/api/pydvl/parallel/futures/index.html
+++ b/devel/api/pydvl/parallel/futures/index.html
@@ -3239,7 +3239,7 @@ <h2 id="pydvl.parallel.futures.init_executor" class="doc doc-heading">
 <a href="#pydvl.parallel.futures.init_executor" class="headerlink" title="Permanent link">&para;</a></h2>
 <div class="language-python doc-signature highlight"><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="n">init_executor</span><span class="p">(</span>
 </span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a>    <span class="n">max_workers</span><span class="p">:</span> <span class="n"><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a></span><span class="p">[</span><span class="n"><a class="autorefs autorefs-external" href="https://docs.python.org/3/library/functions.html#int">int</a></span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a>    <span class="n">config</span><span class="p">:</span> <span class="n"><a class="autorefs autorefs-internal" title="pydvl.parallel.config.ParallelConfig" href="../config/#pydvl.parallel.config.ParallelConfig">ParallelConfig</a></span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
+</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a>    <span class="n">config</span><span class="p">:</span> <span class="n"><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a></span><span class="p">[</span><span class="n"><a class="autorefs autorefs-internal" title="pydvl.parallel.config.ParallelConfig" href="../config/#pydvl.parallel.config.ParallelConfig">ParallelConfig</a></span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 </span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a>    <span class="o">**</span><span class="n">kwargs</span>
 </span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n"><a class="autorefs autorefs-external" title="typing.Generator" href="https://docs.python.org/3/library/typing.html#typing.Generator">Generator</a></span><span class="p">[</span><span class="n"><a class="autorefs autorefs-external" title="concurrent.futures.Executor" href="https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Executor">Executor</a></span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">]</span>
 </span></code></pre></div>
@@ -3286,11 +3286,11 @@ <h2 id="pydvl.parallel.futures.init_executor" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-internal" title="pydvl.parallel.config.ParallelConfig" href="../config/#pydvl.parallel.config.ParallelConfig">ParallelConfig</a></code>
+                    <code><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a>[<a class="autorefs autorefs-internal" title="pydvl.parallel.config.ParallelConfig" href="../config/#pydvl.parallel.config.ParallelConfig">ParallelConfig</a>]</code>
                 </span>
                 <span class="doc-param-default">
                   <b>DEFAULT:</b> 
-                    <code><a class="autorefs autorefs-internal" title="pydvl.parallel.config.ParallelConfig" href="../config/#pydvl.parallel.config.ParallelConfig">ParallelConfig</a>()</code>
+                    <code>None</code>
                 </span>
             </p>
           </td>
@@ -3372,7 +3372,11 @@ <h2 id="pydvl.parallel.futures.init_executor" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-55">55</a></span>
 <span class="normal"><a href="#__codelineno-0-56">56</a></span>
 <span class="normal"><a href="#__codelineno-0-57">57</a></span>
-<span class="normal"><a href="#__codelineno-0-58">58</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-16"><a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="nd">@contextmanager</span>
+<span class="normal"><a href="#__codelineno-0-58">58</a></span>
+<span class="normal"><a href="#__codelineno-0-59">59</a></span>
+<span class="normal"><a href="#__codelineno-0-60">60</a></span>
+<span class="normal"><a href="#__codelineno-0-61">61</a></span>
+<span class="normal"><a href="#__codelineno-0-62">62</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-16"><a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="nd">@contextmanager</span>
 </span><span id="__span-0-17"><a id="__codelineno-0-17" name="__codelineno-0-17"></a><span class="nd">@deprecated</span><span class="p">(</span>
 </span><span id="__span-0-18"><a id="__codelineno-0-18" name="__codelineno-0-18"></a>    <span class="n">target</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
 </span><span id="__span-0-19"><a id="__codelineno-0-19" name="__codelineno-0-19"></a>    <span class="n">deprecated_in</span><span class="o">=</span><span class="s2">&quot;0.9.0&quot;</span><span class="p">,</span>
@@ -3380,7 +3384,7 @@ <h2 id="pydvl.parallel.futures.init_executor" class="doc doc-heading">
 </span><span id="__span-0-21"><a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="p">)</span>
 </span><span id="__span-0-22"><a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="k">def</span> <span class="nf">init_executor</span><span class="p">(</span>
 </span><span id="__span-0-23"><a id="__codelineno-0-23" name="__codelineno-0-23"></a>    <span class="n">max_workers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-24"><a id="__codelineno-0-24" name="__codelineno-0-24"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
+</span><span id="__span-0-24"><a id="__codelineno-0-24" name="__codelineno-0-24"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ParallelConfig</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 </span><span id="__span-0-25"><a id="__codelineno-0-25" name="__codelineno-0-25"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
 </span><span id="__span-0-26"><a id="__codelineno-0-26" name="__codelineno-0-26"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Generator</span><span class="p">[</span><span class="n">Executor</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">]:</span>
 </span><span id="__span-0-27"><a id="__codelineno-0-27" name="__codelineno-0-27"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Initializes a futures executor for the given parallel configuration.</span>
@@ -3409,12 +3413,16 @@ <h2 id="pydvl.parallel.futures.init_executor" class="doc doc-heading">
 </span><span id="__span-0-50"><a id="__codelineno-0-50" name="__codelineno-0-50"></a><span class="sd">        assert results == [1, 2, 3, 4, 5]</span>
 </span><span id="__span-0-51"><a id="__codelineno-0-51" name="__codelineno-0-51"></a><span class="sd">        ```</span>
 </span><span id="__span-0-52"><a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-53"><a id="__codelineno-0-53" name="__codelineno-0-53"></a>    <span class="k">try</span><span class="p">:</span>
-</span><span id="__span-0-54"><a id="__codelineno-0-54" name="__codelineno-0-54"></a>        <span class="bp">cls</span> <span class="o">=</span> <span class="n">ParallelBackend</span><span class="o">.</span><span class="n">BACKENDS</span><span class="p">[</span><span class="n">config</span><span class="o">.</span><span class="n">backend</span><span class="p">]</span>
-</span><span id="__span-0-55"><a id="__codelineno-0-55" name="__codelineno-0-55"></a>        <span class="k">with</span> <span class="bp">cls</span><span class="o">.</span><span class="n">executor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="n">max_workers</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-</span><span id="__span-0-56"><a id="__codelineno-0-56" name="__codelineno-0-56"></a>            <span class="k">yield</span> <span class="n">e</span>
-</span><span id="__span-0-57"><a id="__codelineno-0-57" name="__codelineno-0-57"></a>    <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
-</span><span id="__span-0-58"><a id="__codelineno-0-58" name="__codelineno-0-58"></a>        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Unexpected parallel backend </span><span class="si">{</span><span class="n">config</span><span class="o">.</span><span class="n">backend</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+</span><span id="__span-0-53"><a id="__codelineno-0-53" name="__codelineno-0-53"></a>
+</span><span id="__span-0-54"><a id="__codelineno-0-54" name="__codelineno-0-54"></a>    <span class="k">if</span> <span class="n">config</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-55"><a id="__codelineno-0-55" name="__codelineno-0-55"></a>        <span class="n">config</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">()</span>
+</span><span id="__span-0-56"><a id="__codelineno-0-56" name="__codelineno-0-56"></a>
+</span><span id="__span-0-57"><a id="__codelineno-0-57" name="__codelineno-0-57"></a>    <span class="k">try</span><span class="p">:</span>
+</span><span id="__span-0-58"><a id="__codelineno-0-58" name="__codelineno-0-58"></a>        <span class="bp">cls</span> <span class="o">=</span> <span class="n">ParallelBackend</span><span class="o">.</span><span class="n">BACKENDS</span><span class="p">[</span><span class="n">config</span><span class="o">.</span><span class="n">backend</span><span class="p">]</span>
+</span><span id="__span-0-59"><a id="__codelineno-0-59" name="__codelineno-0-59"></a>        <span class="k">with</span> <span class="bp">cls</span><span class="o">.</span><span class="n">executor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="n">max_workers</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+</span><span id="__span-0-60"><a id="__codelineno-0-60" name="__codelineno-0-60"></a>            <span class="k">yield</span> <span class="n">e</span>
+</span><span id="__span-0-61"><a id="__codelineno-0-61" name="__codelineno-0-61"></a>    <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+</span><span id="__span-0-62"><a id="__codelineno-0-62" name="__codelineno-0-62"></a>        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Unexpected parallel backend </span><span class="si">{</span><span class="n">config</span><span class="o">.</span><span class="n">backend</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3450,7 +3458,7 @@ <h2 id="pydvl.parallel.futures.init_executor" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3460,7 +3468,7 @@ <h2 id="pydvl.parallel.futures.init_executor" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/parallel/futures/ray/index.html b/devel/api/pydvl/parallel/futures/ray/index.html
index ae7d1326b..70485d299 100644
--- a/devel/api/pydvl/parallel/futures/ray/index.html
+++ b/devel/api/pydvl/parallel/futures/ray/index.html
@@ -3961,7 +3961,7 @@ <h3 id="pydvl.parallel.futures.ray.RayExecutor.__exit__" class="doc doc-heading"
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3971,7 +3971,7 @@ <h3 id="pydvl.parallel.futures.ray.RayExecutor.__exit__" class="doc doc-heading"
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/parallel/index.html b/devel/api/pydvl/parallel/index.html
index 8111a4814..f942f1898 100644
--- a/devel/api/pydvl/parallel/index.html
+++ b/devel/api/pydvl/parallel/index.html
@@ -3277,7 +3277,7 @@ <h1 id="pydvl.parallel" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3287,7 +3287,7 @@ <h1 id="pydvl.parallel" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/parallel/map_reduce/index.html b/devel/api/pydvl/parallel/map_reduce/index.html
index fe995e39f..6cb96f86f 100644
--- a/devel/api/pydvl/parallel/map_reduce/index.html
+++ b/devel/api/pydvl/parallel/map_reduce/index.html
@@ -3860,7 +3860,7 @@ <h3 id="pydvl.parallel.map_reduce.MapReduceJob.__call__" class="doc doc-heading"
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3870,7 +3870,7 @@ <h3 id="pydvl.parallel.map_reduce.MapReduceJob.__call__" class="doc doc-heading"
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/reporting/index.html b/devel/api/pydvl/reporting/index.html
index ba32171ee..4bd62b8c6 100644
--- a/devel/api/pydvl/reporting/index.html
+++ b/devel/api/pydvl/reporting/index.html
@@ -3237,7 +3237,7 @@ <h1 id="pydvl.reporting" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3247,7 +3247,7 @@ <h1 id="pydvl.reporting" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/reporting/plots/index.html b/devel/api/pydvl/reporting/plots/index.html
index b7e8c16c2..ed8f7fb80 100644
--- a/devel/api/pydvl/reporting/plots/index.html
+++ b/devel/api/pydvl/reporting/plots/index.html
@@ -5081,7 +5081,7 @@ <h2 id="pydvl.reporting.plots.plot_influence_distribution_by_label" class="doc d
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -5091,7 +5091,7 @@ <h2 id="pydvl.reporting.plots.plot_influence_distribution_by_label" class="doc d
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/reporting/scores/index.html b/devel/api/pydvl/reporting/scores/index.html
index fffbd9771..2e15ea10c 100644
--- a/devel/api/pydvl/reporting/scores/index.html
+++ b/devel/api/pydvl/reporting/scores/index.html
@@ -3533,7 +3533,7 @@ <h2 id="pydvl.reporting.scores.compute_removal_score" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3543,7 +3543,7 @@ <h2 id="pydvl.reporting.scores.compute_removal_score" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/caching/base/index.html b/devel/api/pydvl/utils/caching/base/index.html
index 33c8819f8..5daba2224 100644
--- a/devel/api/pydvl/utils/caching/base/index.html
+++ b/devel/api/pydvl/utils/caching/base/index.html
@@ -4452,7 +4452,7 @@ <h3 id="pydvl.utils.caching.base.CachedFunc.__call__" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -4462,7 +4462,7 @@ <h3 id="pydvl.utils.caching.base.CachedFunc.__call__" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/caching/config/index.html b/devel/api/pydvl/utils/caching/config/index.html
index d0708bd0d..f6eee5c96 100644
--- a/devel/api/pydvl/utils/caching/config/index.html
+++ b/devel/api/pydvl/utils/caching/config/index.html
@@ -3493,7 +3493,7 @@ <h2 id="pydvl.utils.caching.config.CachedFuncConfig" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3503,7 +3503,7 @@ <h2 id="pydvl.utils.caching.config.CachedFuncConfig" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/caching/disk/index.html b/devel/api/pydvl/utils/caching/disk/index.html
index 35481cce1..24e705ed3 100644
--- a/devel/api/pydvl/utils/caching/disk/index.html
+++ b/devel/api/pydvl/utils/caching/disk/index.html
@@ -4001,7 +4001,7 @@ <h3 id="pydvl.utils.caching.disk.DiskCacheBackend.combine_hashes" class="doc doc
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -4011,7 +4011,7 @@ <h3 id="pydvl.utils.caching.disk.DiskCacheBackend.combine_hashes" class="doc doc
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/caching/index.html b/devel/api/pydvl/utils/caching/index.html
index 6ea579ded..ea6428086 100644
--- a/devel/api/pydvl/utils/caching/index.html
+++ b/devel/api/pydvl/utils/caching/index.html
@@ -3366,7 +3366,7 @@ <h2 id="pydvl.utils.caching--unexpected-cache-misses">Unexpected cache misses<a
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3376,7 +3376,7 @@ <h2 id="pydvl.utils.caching--unexpected-cache-misses">Unexpected cache misses<a
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/caching/memcached/index.html b/devel/api/pydvl/utils/caching/memcached/index.html
index 15e0a3c8f..68f060ee7 100644
--- a/devel/api/pydvl/utils/caching/memcached/index.html
+++ b/devel/api/pydvl/utils/caching/memcached/index.html
@@ -4303,7 +4303,7 @@ <h3 id="pydvl.utils.caching.memcached.MemcachedCacheBackend.__setstate__" class=
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -4313,7 +4313,7 @@ <h3 id="pydvl.utils.caching.memcached.MemcachedCacheBackend.__setstate__" class=
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/caching/memory/index.html b/devel/api/pydvl/utils/caching/memory/index.html
index 138637b87..9fac1a613 100644
--- a/devel/api/pydvl/utils/caching/memory/index.html
+++ b/devel/api/pydvl/utils/caching/memory/index.html
@@ -3911,7 +3911,7 @@ <h3 id="pydvl.utils.caching.memory.InMemoryCacheBackend.combine_hashes" class="d
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3921,7 +3921,7 @@ <h3 id="pydvl.utils.caching.memory.InMemoryCacheBackend.combine_hashes" class="d
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/config/index.html b/devel/api/pydvl/utils/config/index.html
index 5f61c9f2b..6814cf061 100644
--- a/devel/api/pydvl/utils/config/index.html
+++ b/devel/api/pydvl/utils/config/index.html
@@ -3662,7 +3662,7 @@ <h2 id="pydvl.utils.config.CachedFuncConfig" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3672,7 +3672,7 @@ <h2 id="pydvl.utils.config.CachedFuncConfig" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/dataset/index.html b/devel/api/pydvl/utils/dataset/index.html
index 35f480a73..d11d3f5c1 100644
--- a/devel/api/pydvl/utils/dataset/index.html
+++ b/devel/api/pydvl/utils/dataset/index.html
@@ -6689,7 +6689,7 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.from_dataset" class="doc doc-heading"
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -6699,7 +6699,7 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.from_dataset" class="doc doc-heading"
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/functional/index.html b/devel/api/pydvl/utils/functional/index.html
index 7705b9c01..e17682cf1 100644
--- a/devel/api/pydvl/utils/functional/index.html
+++ b/devel/api/pydvl/utils/functional/index.html
@@ -3649,7 +3649,7 @@ <h2 id="pydvl.utils.functional.maybe_add_argument" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3659,7 +3659,7 @@ <h2 id="pydvl.utils.functional.maybe_add_argument" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/index.html b/devel/api/pydvl/utils/index.html
index 95af413ef..df402c1f8 100644
--- a/devel/api/pydvl/utils/index.html
+++ b/devel/api/pydvl/utils/index.html
@@ -3237,7 +3237,7 @@ <h1 id="pydvl.utils" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3247,7 +3247,7 @@ <h1 id="pydvl.utils" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/numeric/index.html b/devel/api/pydvl/utils/numeric/index.html
index efa4707c6..24c7c0999 100644
--- a/devel/api/pydvl/utils/numeric/index.html
+++ b/devel/api/pydvl/utils/numeric/index.html
@@ -5118,7 +5118,7 @@ <h2 id="pydvl.utils.numeric.top_k_value_accuracy" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -5128,7 +5128,7 @@ <h2 id="pydvl.utils.numeric.top_k_value_accuracy" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/progress/index.html b/devel/api/pydvl/utils/progress/index.html
index 7a2ccd3f6..8b9c295be 100644
--- a/devel/api/pydvl/utils/progress/index.html
+++ b/devel/api/pydvl/utils/progress/index.html
@@ -3518,7 +3518,7 @@ <h2 id="pydvl.utils.progress.log_duration" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3528,7 +3528,7 @@ <h2 id="pydvl.utils.progress.log_duration" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/score/index.html b/devel/api/pydvl/utils/score/index.html
index 52841a730..77c063991 100644
--- a/devel/api/pydvl/utils/score/index.html
+++ b/devel/api/pydvl/utils/score/index.html
@@ -3849,7 +3849,7 @@ <h2 id="pydvl.utils.score.compose_score" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3859,7 +3859,7 @@ <h2 id="pydvl.utils.score.compose_score" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/status/index.html b/devel/api/pydvl/utils/status/index.html
index 11428543d..3fc80ba23 100644
--- a/devel/api/pydvl/utils/status/index.html
+++ b/devel/api/pydvl/utils/status/index.html
@@ -3433,7 +3433,7 @@ <h4 id="pydvl.utils.status.Status--boolean-casting">Boolean casting<a class="hea
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3443,7 +3443,7 @@ <h4 id="pydvl.utils.status.Status--boolean-casting">Boolean casting<a class="hea
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/types/index.html b/devel/api/pydvl/utils/types/index.html
index f5e642340..f14879c97 100644
--- a/devel/api/pydvl/utils/types/index.html
+++ b/devel/api/pydvl/utils/types/index.html
@@ -3853,7 +3853,7 @@ <h2 id="pydvl.utils.types.ensure_seed_sequence" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3863,7 +3863,7 @@ <h2 id="pydvl.utils.types.ensure_seed_sequence" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/utils/utility/index.html b/devel/api/pydvl/utils/utility/index.html
index 7a46778f9..b7c33f012 100644
--- a/devel/api/pydvl/utils/utility/index.html
+++ b/devel/api/pydvl/utils/utility/index.html
@@ -4120,7 +4120,7 @@ <h3 id="pydvl.utils.utility.DataUtilityLearning.data" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -4130,7 +4130,7 @@ <h3 id="pydvl.utils.utility.DataUtilityLearning.data" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/games/index.html b/devel/api/pydvl/value/games/index.html
index 91aaf8b6d..0c83565d7 100644
--- a/devel/api/pydvl/value/games/index.html
+++ b/devel/api/pydvl/value/games/index.html
@@ -5751,7 +5751,7 @@ <h2 id="pydvl.value.games.MinerGame" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -5761,7 +5761,7 @@ <h2 id="pydvl.value.games.MinerGame" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/index.html b/devel/api/pydvl/value/index.html
index 64de49720..f7557d088 100644
--- a/devel/api/pydvl/value/index.html
+++ b/devel/api/pydvl/value/index.html
@@ -3242,7 +3242,7 @@ <h1 id="pydvl.value" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3252,7 +3252,7 @@ <h1 id="pydvl.value" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/least_core/common/index.html b/devel/api/pydvl/value/least_core/common/index.html
index cbffe0bdf..810a6db5c 100644
--- a/devel/api/pydvl/value/least_core/common/index.html
+++ b/devel/api/pydvl/value/least_core/common/index.html
@@ -3933,7 +3933,7 @@ <h2 id="pydvl.value.least_core.common.lc_solve_problems" class="doc doc-heading"
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3943,7 +3943,7 @@ <h2 id="pydvl.value.least_core.common.lc_solve_problems" class="doc doc-heading"
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/least_core/index.html b/devel/api/pydvl/value/least_core/index.html
index 1b8b9ea52..b6b2b75b6 100644
--- a/devel/api/pydvl/value/least_core/index.html
+++ b/devel/api/pydvl/value/least_core/index.html
@@ -3649,7 +3649,7 @@ <h2 id="pydvl.value.least_core.compute_least_core_values" class="doc doc-heading
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3659,7 +3659,7 @@ <h2 id="pydvl.value.least_core.compute_least_core_values" class="doc doc-heading
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/least_core/montecarlo/index.html b/devel/api/pydvl/value/least_core/montecarlo/index.html
index 4bf6b84b6..1b49367d5 100644
--- a/devel/api/pydvl/value/least_core/montecarlo/index.html
+++ b/devel/api/pydvl/value/least_core/montecarlo/index.html
@@ -3908,7 +3908,7 @@ <h2 id="pydvl.value.least_core.montecarlo.mclc_prepare_problem" class="doc doc-h
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3918,7 +3918,7 @@ <h2 id="pydvl.value.least_core.montecarlo.mclc_prepare_problem" class="doc doc-h
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/least_core/naive/index.html b/devel/api/pydvl/value/least_core/naive/index.html
index 3f7163796..df58a75b8 100644
--- a/devel/api/pydvl/value/least_core/naive/index.html
+++ b/devel/api/pydvl/value/least_core/naive/index.html
@@ -3677,7 +3677,7 @@ <h2 id="pydvl.value.least_core.naive.lc_prepare_problem" class="doc doc-heading"
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3687,7 +3687,7 @@ <h2 id="pydvl.value.least_core.naive.lc_prepare_problem" class="doc doc-heading"
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/loo/index.html b/devel/api/pydvl/value/loo/index.html
index 0e7c73a37..4f22d205c 100644
--- a/devel/api/pydvl/value/loo/index.html
+++ b/devel/api/pydvl/value/loo/index.html
@@ -3239,7 +3239,7 @@ <h1 id="pydvl.value.loo" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3249,7 +3249,7 @@ <h1 id="pydvl.value.loo" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/loo/loo/index.html b/devel/api/pydvl/value/loo/loo/index.html
index be76bdcae..e7e5d62bd 100644
--- a/devel/api/pydvl/value/loo/loo/index.html
+++ b/devel/api/pydvl/value/loo/loo/index.html
@@ -3666,7 +3666,7 @@ <h2 id="pydvl.value.loo.loo.compute_loo" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3676,7 +3676,7 @@ <h2 id="pydvl.value.loo.loo.compute_loo" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/oob/index.html b/devel/api/pydvl/value/oob/index.html
index d1b4fc9a5..f11215d10 100644
--- a/devel/api/pydvl/value/oob/index.html
+++ b/devel/api/pydvl/value/oob/index.html
@@ -3239,7 +3239,7 @@ <h1 id="pydvl.value.oob" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3249,7 +3249,7 @@ <h1 id="pydvl.value.oob" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/oob/oob/index.html b/devel/api/pydvl/value/oob/oob/index.html
index 0eac2880a..5ea716ef0 100644
--- a/devel/api/pydvl/value/oob/oob/index.html
+++ b/devel/api/pydvl/value/oob/oob/index.html
@@ -4037,7 +4037,7 @@ <h2 id="pydvl.value.oob.oob.neg_l2_distance" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -4047,7 +4047,7 @@ <h2 id="pydvl.value.oob.oob.neg_l2_distance" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/result/index.html b/devel/api/pydvl/value/result/index.html
index b9e654d96..cc0148115 100644
--- a/devel/api/pydvl/value/result/index.html
+++ b/devel/api/pydvl/value/result/index.html
@@ -6159,7 +6159,7 @@ <h3 id="pydvl.value.result.ValuationResult.zeros" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -6169,7 +6169,7 @@ <h3 id="pydvl.value.result.ValuationResult.zeros" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/sampler/index.html b/devel/api/pydvl/value/sampler/index.html
index ad0268d0d..a7ac47b17 100644
--- a/devel/api/pydvl/value/sampler/index.html
+++ b/devel/api/pydvl/value/sampler/index.html
@@ -5772,7 +5772,7 @@ <h3 id="pydvl.value.sampler.RandomHierarchicalSampler.__len__" class="doc doc-he
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -5782,7 +5782,7 @@ <h3 id="pydvl.value.sampler.RandomHierarchicalSampler.__len__" class="doc doc-he
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/semivalues/index.html b/devel/api/pydvl/value/semivalues/index.html
index 6b9c824e1..febc189b2 100644
--- a/devel/api/pydvl/value/semivalues/index.html
+++ b/devel/api/pydvl/value/semivalues/index.html
@@ -7086,7 +7086,7 @@ <h2 id="pydvl.value.semivalues.compute_semivalues" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -7096,7 +7096,7 @@ <h2 id="pydvl.value.semivalues.compute_semivalues" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/shapley/classwise/index.html b/devel/api/pydvl/value/shapley/classwise/index.html
index c9b4ba45c..59b5dbeb0 100644
--- a/devel/api/pydvl/value/shapley/classwise/index.html
+++ b/devel/api/pydvl/value/shapley/classwise/index.html
@@ -4557,7 +4557,7 @@ <h2 id="pydvl.value.shapley.classwise.compute_classwise_shapley_values" class="d
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -4567,7 +4567,7 @@ <h2 id="pydvl.value.shapley.classwise.compute_classwise_shapley_values" class="d
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/shapley/common/index.html b/devel/api/pydvl/value/shapley/common/index.html
index 61a9d1d62..8a5ef867d 100644
--- a/devel/api/pydvl/value/shapley/common/index.html
+++ b/devel/api/pydvl/value/shapley/common/index.html
@@ -3807,7 +3807,7 @@ <h2 id="pydvl.value.shapley.common.compute_shapley_values" class="doc doc-headin
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3817,7 +3817,7 @@ <h2 id="pydvl.value.shapley.common.compute_shapley_values" class="doc doc-headin
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/shapley/gt/index.html b/devel/api/pydvl/value/shapley/gt/index.html
index 937cba156..e62398846 100644
--- a/devel/api/pydvl/value/shapley/gt/index.html
+++ b/devel/api/pydvl/value/shapley/gt/index.html
@@ -4101,7 +4101,7 @@ <h2 id="pydvl.value.shapley.gt.group_testing_shapley" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -4111,7 +4111,7 @@ <h2 id="pydvl.value.shapley.gt.group_testing_shapley" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/shapley/index.html b/devel/api/pydvl/value/shapley/index.html
index 23ac59d68..e073eb062 100644
--- a/devel/api/pydvl/value/shapley/index.html
+++ b/devel/api/pydvl/value/shapley/index.html
@@ -3247,7 +3247,7 @@ <h1 id="pydvl.value.shapley" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3257,7 +3257,7 @@ <h1 id="pydvl.value.shapley" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/shapley/knn/index.html b/devel/api/pydvl/value/shapley/knn/index.html
index e90b8265d..f9131ee41 100644
--- a/devel/api/pydvl/value/shapley/knn/index.html
+++ b/devel/api/pydvl/value/shapley/knn/index.html
@@ -3609,7 +3609,7 @@ <h2 id="pydvl.value.shapley.knn.knn_shapley" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3619,7 +3619,7 @@ <h2 id="pydvl.value.shapley.knn.knn_shapley" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/shapley/montecarlo/index.html b/devel/api/pydvl/value/shapley/montecarlo/index.html
index 779ff90f4..0690b4b67 100644
--- a/devel/api/pydvl/value/shapley/montecarlo/index.html
+++ b/devel/api/pydvl/value/shapley/montecarlo/index.html
@@ -4227,7 +4227,7 @@ <h2 id="pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley" class="
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -4237,7 +4237,7 @@ <h2 id="pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley" class="
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/shapley/naive/index.html b/devel/api/pydvl/value/shapley/naive/index.html
index 538b5465c..8b324a35a 100644
--- a/devel/api/pydvl/value/shapley/naive/index.html
+++ b/devel/api/pydvl/value/shapley/naive/index.html
@@ -3854,7 +3854,7 @@ <h2 id="pydvl.value.shapley.naive.combinatorial_exact_shapley" class="doc doc-he
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3864,7 +3864,7 @@ <h2 id="pydvl.value.shapley.naive.combinatorial_exact_shapley" class="doc doc-he
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/shapley/owen/index.html b/devel/api/pydvl/value/shapley/owen/index.html
index c80517a8e..4deb30594 100644
--- a/devel/api/pydvl/value/shapley/owen/index.html
+++ b/devel/api/pydvl/value/shapley/owen/index.html
@@ -3904,7 +3904,7 @@ <h2 id="pydvl.value.shapley.owen.owen_sampling_shapley" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3914,7 +3914,7 @@ <h2 id="pydvl.value.shapley.owen.owen_sampling_shapley" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/shapley/truncated/index.html b/devel/api/pydvl/value/shapley/truncated/index.html
index a84b1e925..26fe5bf66 100644
--- a/devel/api/pydvl/value/shapley/truncated/index.html
+++ b/devel/api/pydvl/value/shapley/truncated/index.html
@@ -4677,7 +4677,7 @@ <h3 id="pydvl.value.shapley.truncated.BootstrapTruncation.__call__" class="doc d
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -4687,7 +4687,7 @@ <h3 id="pydvl.value.shapley.truncated.BootstrapTruncation.__call__" class="doc d
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/shapley/types/index.html b/devel/api/pydvl/value/shapley/types/index.html
index 5e73e473e..585e6e7be 100644
--- a/devel/api/pydvl/value/shapley/types/index.html
+++ b/devel/api/pydvl/value/shapley/types/index.html
@@ -3349,7 +3349,7 @@ <h2 id="pydvl.value.shapley.types.ShapleyMode" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -3359,7 +3359,7 @@ <h2 id="pydvl.value.shapley.types.ShapleyMode" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/api/pydvl/value/stopping/index.html b/devel/api/pydvl/value/stopping/index.html
index 884c9c8f8..4149c7f77 100644
--- a/devel/api/pydvl/value/stopping/index.html
+++ b/devel/api/pydvl/value/stopping/index.html
@@ -6316,7 +6316,7 @@ <h2 id="pydvl.value.stopping.make_criterion" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
@@ -6326,7 +6326,7 @@ <h2 id="pydvl.value.stopping.make_criterion" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-12</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2024-04-17</span>
   </span>
 
     
diff --git a/devel/search/search_index.json b/devel/search/search_index.json
index d7b301679..1ad874367 100644
--- a/devel/search/search_index.json
+++ b/devel/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"The python library for data valuation","text":"<p>pyDVL collects algorithms for data valuation and influence function computation. For the full list see Methods. It supports out-of-core and distributed computation, as well as local or distributed caching of results.</p> <p>If you're a first time user of pyDVL, we recommend you to go through Getting started.</p> <ul> <li> <p> Getting started</p> <p>Steps to install and requirements</p> </li> <li> <p> Example gallery</p> <p>Notebooks with worked-out examples of data valuation and influence functions</p> </li> <li> <p> Data valuation</p> <p>Basics of data valuation and description of the main algorithms</p> </li> <li> <p> Influence Function</p> <p>An introduction to the influence function and its computation with pyDVL</p> </li> <li> <p> Supported methods</p> <p>List of all methods implemented with references.</p> </li> <li> <p> API Reference</p> <p>Full documentation of the API</p> </li> </ul>"},{"location":"CHANGELOG/","title":"Changelog","text":""},{"location":"CHANGELOG/#090-new-methods-better-docs-and-bugfixes","title":"0.9.0 \ud83c\udd95 New methods, better docs and bugfixes \ud83d\udcda\ud83d\udc1e","text":""},{"location":"CHANGELOG/#added","title":"Added","text":"<ul> <li>New method <code>MSR Banzhaf</code> with accompanying notebook, and new stopping   criterion <code>RankCorrelation</code> PR #520</li> <li>New method: <code>NystroemSketchInfluence</code> PR #504</li> <li>New preconditioned block variant of conjugate gradient   PR #507</li> <li>Improvements to documentation: fixes, links, text, example gallery, LFS and   more PR #532,    PR #543</li> <li>Glossary of data valuation and influence terms in the documentation   [PR #537](https://github.com/aai-institute/pyDVL/pull/537</li> <li>Documentation about writing notes for new features, changes or deprecations   PR #557</li> </ul>"},{"location":"CHANGELOG/#fixed","title":"Fixed","text":"<ul> <li>Bug in <code>LissaInfluence</code>, when not using CPU device   PR #495</li> <li>Memory issue with <code>CgInfluence</code> and <code>ArnoldiInfluence</code> PR #498</li> <li>Raising specific error message with install instruction, when trying to load    <code>pydvl.utils.cache.memcached</code> without <code>pymemcache</code> installed.   If <code>pymemcache</code> is available, all symbols from <code>pydvl.utils.cache.memcached</code>    are available through <code>pydvl.utils.cache</code> PR #509 </li> </ul>"},{"location":"CHANGELOG/#changed","title":"Changed","text":"<ul> <li>Add property <code>model_dtype</code> to instances of type <code>TorchInfluenceFunctionModel</code></li> <li>Bump versions of CI actions to avoid warnings   PR #502</li> <li>Add Python Version 3.11 to supported versions   PR #510</li> <li>Documentation improvements and cleanup   PR #521,   PR #522</li> <li>Simplified parallel backend configuration   PR #549</li> </ul>"},{"location":"CHANGELOG/#081-new-method-and-notebook-games-with-exact-shapley-values-bug-fixes-and-cleanup","title":"0.8.1 - \ud83c\udd95 \ud83c\udfd7  New method and notebook, Games with exact shapley values, bug fixes and cleanup","text":""},{"location":"CHANGELOG/#added_1","title":"Added","text":"<ul> <li>Implement new method: <code>EkfacInfluence</code> PR #451</li> <li>New notebook to showcase ekfac for LLMs   PR #483</li> <li>Implemented exact games in Castro et al. 2009 and 2017   PR #341</li> </ul>"},{"location":"CHANGELOG/#fixed_1","title":"Fixed","text":"<ul> <li>Bug in using <code>DaskInfluenceCalcualator</code> with <code>TorchnumpyConverter</code>   for single dimensional arrays    PR #485</li> <li>Fix implementations of <code>to</code> methods of <code>TorchInfluenceFunctionModel</code>    implementations PR #487</li> <li>Fixed bug with checking for converged values in semivalues   PR #341</li> </ul>"},{"location":"CHANGELOG/#changed_1","title":"Changed","text":"<ul> <li>Add applications of data valuation section, display examples more prominently,   make all sections visible in table of contents, use mkdocs material cards   in the home page PR #492</li> </ul>"},{"location":"CHANGELOG/#080-new-interfaces-scaling-computation-bug-fixes-and-improvements","title":"0.8.0 - \ud83c\udd95 New interfaces, scaling computation, bug fixes and improvements \ud83c\udf81","text":""},{"location":"CHANGELOG/#added_2","title":"Added","text":"<ul> <li>New cache backends: InMemoryCacheBackend and DiskCacheBackend   PR #458</li> <li>New influence function interface <code>InfluenceFunctionModel</code></li> <li>Data parallel computation with <code>DaskInfluenceCalculator</code> PR #26</li> <li>Sequential batch-wise computation and write to disk with    <code>SequentialInfluenceCalculator</code> PR #377</li> <li>Adapt notebooks to new influence abstractions   PR #430</li> </ul>"},{"location":"CHANGELOG/#changed_2","title":"Changed","text":"<ul> <li>Refactor and simplify caching implementation    PR #458</li> <li>Simplify display of computation progress   PR #466</li> <li>Improve readme and explain better the examples   PR #465</li> <li>Simplify and improve tests, add CodeCov code coverage   PR #429</li> <li>Breaking Changes</li> <li>Removed <code>compute_influences</code> and all related code.     Replaced by new <code>InfluenceFunctionModel</code> interface. Removed modules:<ul> <li>influence.general</li> <li>influence.inversion</li> <li>influence.twice_differentiable</li> <li>influence.torch.torch_differentiable</li> </ul> </li> </ul>"},{"location":"CHANGELOG/#fixed_2","title":"Fixed","text":"<ul> <li>Import bug in README PR #457</li> </ul>"},{"location":"CHANGELOG/#071-new-methods-bug-fixes-and-improvements-for-local-tests","title":"0.7.1 - \ud83c\udd95 New methods, bug fixes and improvements for local tests \ud83d\udc1e\ud83e\uddea","text":""},{"location":"CHANGELOG/#added_3","title":"Added","text":"<ul> <li>New method: Class-wise Shapley values   PR #338</li> <li>New method: Data-OOB by @BastienZim    PR #426,    PR $431</li> <li>Added <code>AntitheticPermutationSampler</code> PR #439</li> <li>Faster semi-value computation with per-index check of stopping criteria (optional)   PR #437</li> </ul>"},{"location":"CHANGELOG/#fixed_3","title":"Fixed","text":"<ul> <li>Fix initialization of <code>data_names</code> in <code>ValuationResult.zeros()</code> PR #443</li> </ul>"},{"location":"CHANGELOG/#changed_3","title":"Changed","text":"<ul> <li>No longer using docker within tests to start a memcached server   PR #444</li> <li>Using pytest-xdist for faster local tests   PR #440</li> <li>Improvements and fixes to notebooks   PR #436</li> <li>Refactoring of parallel module. Old imports will stop working in v0.9.0   PR #421</li> </ul>"},{"location":"CHANGELOG/#070-documentation-and-if-overhaul-new-methods-and-bug-fixes","title":"0.7.0 - \ud83d\udcda\ud83c\udd95 Documentation and IF overhaul, new methods and bug fixes \ud83d\udca5\ud83d\udc1e","text":"<p>This is our first \u03b2 release! We have worked hard to deliver improvements across the board, with a focus on documentation and usability. We have also reworked the internals of the <code>influence</code> module, improved parallelism and handling of randomness.</p>"},{"location":"CHANGELOG/#added_4","title":"Added","text":"<ul> <li>Implemented solving the Hessian equation via spectral low-rank approximation   PR #365</li> <li>Enabled parallel computation for Leave-One-Out values   PR #406</li> <li>Added more abbreviations to documentation   PR #415</li> <li>Added seed to functions from <code>pydvl.utils.numeric</code>, <code>pydvl.value.shapley</code> and   <code>pydvl.value.semivalues</code>. Introduced new type <code>Seed</code> and conversion function    <code>ensure_seed_sequence</code>.   PR #396</li> <li>Added <code>batch_size</code> parameter to <code>compute_banzhaf_semivalues</code>,   <code>compute_beta_shapley_semivalues</code>, <code>compute_shapley_semivalues</code> and   <code>compute_generic_semivalues</code>.   PR #428</li> <li>Added classwise Shapley as proposed by (Schoch et al. 2021)   [https://arxiv.org/abs/2211.06800]   PR #338</li> </ul>"},{"location":"CHANGELOG/#changed_4","title":"Changed","text":"<ul> <li>Replaced sphinx with mkdocs for documentation. Major overhaul of documentation   PR #352</li> <li>Made ray an optional dependency, relying on joblib as default parallel backend   PR #408</li> <li>Decoupled <code>ray.init</code> from <code>ParallelConfig</code> PR #373</li> <li>Breaking Changes</li> <li>Signature change: return information about Hessian inversion from     <code>compute_influence_factors</code> PR #375</li> <li>Major changes to IF interface and functionality. Foundation for a framework     abstraction for IF computation.     PR #278 PR #394</li> <li>Renamed <code>semivalues</code> to <code>compute_generic_semivalues</code> PR #413</li> <li>New <code>joblib</code> backend as default instead of ray. Simplify MapReduceJob.     PR #355</li> <li>Bump torch dependency for influence package to 2.0     PR #365</li> </ul>"},{"location":"CHANGELOG/#fixed_4","title":"Fixed","text":"<ul> <li>Fixes to parallel computation of generic semi-values: properly handle all   samplers and stopping criteria, irrespective of parallel backend.   PR #372</li> <li>Optimises memory usage in IF calculation   PR #375</li> <li>Fix adding valuation results with overlapping indices and different lengths   PR #370</li> <li>Fixed bugs in conjugate gradient and <code>linear_solve</code> PR #358</li> <li>Fix installation of dev requirements for Python3.10   PR #382</li> <li>Improvements to IF documentation   PR #371</li> </ul>"},{"location":"CHANGELOG/#061-bug-fixes-and-small-improvements","title":"0.6.1 - \ud83c\udfd7 Bug fixes and small improvements","text":"<ul> <li>Fix parsing keyword arguments of <code>compute_semivalues</code> dispatch function   PR #333</li> <li>Create new <code>RayExecutor</code> class based on the concurrent.futures API,   use the new class to fix an issue with Truncated Monte Carlo Shapley   (TMCS) starting too many processes and dying, plus other small changes   PR #329</li> <li>Fix creation of GroupedDataset objects using the <code>from_arrays</code>   and <code>from_sklearn</code> class methods    PR #324</li> <li>Fix release job not triggering on CI when a new tag is pushed   PR #331</li> <li>Added alias <code>ApproShapley</code> from Castro et al. 2009 for permutation Shapley   PR #332</li> </ul>"},{"location":"CHANGELOG/#060-new-algorithms-cleanup-and-bug-fixes","title":"0.6.0 - \ud83c\udd95 New algorithms, cleanup and bug fixes \ud83c\udfd7","text":"<ul> <li>Fixes in <code>ValuationResult</code>: bugs around data names, semantics of   <code>empty()</code>, new method <code>zeros()</code> and normalised random values   PR #327</li> <li>New method: Implements generalised semi-values for data valuation,   including Data Banzhaf and Beta Shapley, with configurable sampling strategies   PR #319</li> <li>Adds kwargs parameter to <code>from_array</code> and <code>from_sklearn</code> Dataset and   GroupedDataset class methods   PR #316</li> <li>PEP-561 conformance: added <code>py.typed</code> PR #307</li> <li>Removed default non-negativity constraint on least core subsidy   and added instead a <code>non_negative_subsidy</code> boolean flag.   Renamed <code>options</code> to <code>solver_options</code> and pass it as dict.   Change default least-core solver to SCS with 10000 max_iters.   PR #304</li> <li>Cleanup: removed unnecessary decorator <code>@unpackable</code> PR #233</li> <li>Stopping criteria: fixed problem with <code>StandardError</code> and enable proper   composition of index convergence statuses. Fixed a bug with <code>n_jobs</code> in   <code>truncated_montecarlo_shapley</code>.   PR #300 and   PR #305</li> <li>Shuffling code around to allow for simpler user imports, some cleanup and   documentation fixes.   PR #284</li> <li>Bug fix: Warn instead of raising an error when <code>n_iterations</code>   is less than the size of the dataset in Monte Carlo Least Core   PR #281</li> </ul>"},{"location":"CHANGELOG/#050-fixes-nicer-interfaces-and-more-breaking-changes","title":"0.5.0 - \ud83d\udca5 Fixes, nicer interfaces and... more breaking changes \ud83d\ude12","text":"<ul> <li>Fixed parallel and antithetic Owen sampling for Shapley values. Simplified   and extended tests.   PR #267</li> <li>Added <code>Scorer</code> class for a cleaner interface. Fixed minor bugs around   Group-Testing Shapley, added more tests and switched to cvxpy for the solver.   PR #264</li> <li>Generalised stopping criteria for valuation algorithms. Improved classes   <code>ValuationResult</code> and <code>Status</code> with more operations. Some minor issues fixed.   PR #252</li> <li>Fixed a bug whereby <code>compute_shapley_values</code> would only spawn one process when   using <code>n_jobs=-1</code> and Monte Carlo methods.   PR #270</li> <li>Bugfix in <code>RayParallelBackend</code>: wrong semantics for <code>kwargs</code>.   PR #268</li> <li>Splitting of problem preparation and solution in Least-Core computation.   Umbrella function for LC methods.   PR #257 </li> <li>Operations on <code>ValuationResult</code> and <code>Status</code> and some cleanup   PR #248</li> <li>Bug fix and minor improvements: Fixes bug in TMCS with remote Ray cluster,   raises an error for dummy sequential parallel backend with TMCS, clones model   inside <code>Utility</code> before fitting by default, with flag <code>clone_before_fit</code>    to disable it, catches all warnings in <code>Utility</code> when <code>show_warnings</code> is    <code>False</code>. Adds Miner and Gloves toy games utilities   PR #247</li> </ul>"},{"location":"CHANGELOG/#040-new-algorithms-and-more-breaking-changes","title":"0.4.0 - \ud83c\udfed\ud83d\udca5 New algorithms and more breaking changes","text":"<ul> <li>GH action to mark issues as stale   PR #201</li> <li>Disabled caching of Utility values as well as repeated evaluations by default   PR #211</li> <li>Test and officially support Python version 3.9 and 3.10    PR #208</li> <li>Breaking change: Introduces a class ValuationResult to gather and inspect   results from all valuation algorithms   PR #214</li> <li>Fixes bug in Influence calculation with multidimensional input and adds new   example notebook   PR #195</li> <li>Breaking change: Passes the input to <code>MapReduceJob</code> at initialization,   removes <code>chunkify_inputs</code> argument from <code>MapReduceJob</code>, removes <code>n_runs</code>   argument from <code>MapReduceJob</code>, calls the parallel backend's <code>put()</code> method for   each generated chunk in <code>_chunkify()</code>, renames ParallelConfig's <code>num_workers</code>   attribute to <code>n_local_workers</code>, fixes a bug in <code>MapReduceJob</code>'s chunkification   when <code>n_runs</code> &gt;= <code>n_jobs</code>, and defines a sequential parallel backend to run   all jobs in the current thread   PR #232</li> <li>New method: Implements exact and monte carlo Least Core for data valuation,   adds <code>from_arrays()</code> class method to the <code>Dataset</code> and <code>GroupedDataset</code>   classes, adds <code>extra_values</code> argument to <code>ValuationResult</code>, adds   <code>compute_removal_score()</code> and <code>compute_random_removal_score()</code> helper functions   PR #237</li> <li>New method: Group Testing Shapley for valuation, from Jia et al. 2019 PR #240</li> <li>Fixes bug in ray initialization in <code>RayParallelBackend</code> class   PR #239</li> <li>Implements \"Egalitarian Least Core\", adds cvxpy as a   dependency and uses it instead of scipy as optimizer   PR #243</li> </ul>"},{"location":"CHANGELOG/#030-breaking-changes","title":"0.3.0 - \ud83d\udca5 Breaking changes","text":"<ul> <li>Simplified and fixed powerset sampling and testing   PR #181</li> <li>Simplified and fixed publishing to PyPI from CI   PR #183</li> <li>Fixed bug in release script and updated contributing docs.   PR #184</li> <li>Added Pull Request template   PR #185</li> <li>Modified Pull Request template to automatically link PR to issue   PR ##186</li> <li>First implementation of Owen Sampling, squashed scores, better testing   PR #194</li> <li>Improved documentation on caching, Shapley, caveats of values, bibtex   PR #194</li> <li>Breaking change: Rearranging of modules to accommodate for new methods   PR #194</li> </ul>"},{"location":"CHANGELOG/#020-better-docs","title":"0.2.0 - \ud83d\udcda Better docs","text":"<p>Mostly API documentation and notebooks, plus some bugfixes.</p>"},{"location":"CHANGELOG/#added_5","title":"Added","text":"<p>In PR #161: - Support for $$ math in sphinx docs. - Usage of sphinx extension for external links (introducing new directives like   <code>:gh:</code>, <code>:issue:</code> and <code>:tfl:</code> to construct standardised links to external   resources). - Only update auto-generated documentation files if there are changes. Some   minor additions to <code>update_docs.py</code>. - Parallelization of exact combinatorial Shapley. - Integrated KNN shapley into the main interface <code>compute_shapley_values</code>.</p>"},{"location":"CHANGELOG/#changed_5","title":"Changed","text":"<p>In PR #161: - Improved main docs and Shapley notebooks. Added or fixed many docstrings,   readme and documentation for contributors. Typos, grammar and style in code,   documentation and notebooks. - Internal renaming and rearranging in the parallelization and caching modules.</p>"},{"location":"CHANGELOG/#fixed_5","title":"Fixed","text":"<ul> <li>Bug in random matrix generation   PR #161.</li> <li>Bugs in MapReduceJob's <code>_chunkify</code> and <code>_backpressure</code> methods   PR #176.</li> </ul>"},{"location":"CHANGELOG/#010-first-release","title":"0.1.0 - \ud83c\udf89 first release","text":"<p>This is very first release of pyDVL.</p> <p>It contains:</p> <ul> <li> <p>Data Valuation Methods:</p> </li> <li> <p>Leave-One-Out</p> </li> <li>Influence Functions</li> <li>Shapley:<ul> <li>Exact Permutation and Combinatorial</li> <li>Montecarlo Permutation and Combinatorial</li> <li>Truncated Montecarlo Permutation</li> </ul> </li> <li>Caching of results with Memcached</li> <li>Parallelization of computations with Ray</li> <li>Documentation</li> <li>Notebooks containing examples of different use cases</li> </ul>"},{"location":"CONTRIBUTING/","title":"Contributing to pyDVL","text":"<p>The goal of pyDVL is to be a repository of successful algorithms for the valuation of data, in a broader sense. Contributions are welcome from anyone in the form of pull requests, bug reports and feature requests.</p> <p>We will consider for inclusion any (tested) implementation of an algorithm appearing in a peer-reviewed journal (even if the method does not improve the state of the art, for benchmarking and comparison purposes). We are also open to improvements to the currently implemented methods and other ideas. Please open a ticket with yours.</p> <p>If you are interested in setting up a similar project, consider the template  pymetrius.</p>"},{"location":"CONTRIBUTING/#local-development","title":"Local development","text":"<p>This project uses black to format code and pre-commit to invoke it as a git pre-commit hook. Consider installing any of black's IDE integrations to make your life easier.</p> <p>Run the following to set up the pre-commit git hook to run before pushes:</p> <pre><code>pre-commit install --hook-type pre-push\n</code></pre> <p>Additionally, we use Git LFS for some files like images. Install with</p> <pre><code>git lfs install\n</code></pre>"},{"location":"CONTRIBUTING/#setting-up-your-environment","title":"Setting up your environment","text":"<p>We strongly suggest using some form of virtual environment for working with the library. E.g. with venv:</p> <pre><code>python -m venv ./venv\n. venv/bin/activate  # `venv\\Scripts\\activate` in windows\npip install -r requirements-dev.txt -r requirements-docs.txt\n</code></pre> <p>With conda:</p> <pre><code>conda create -n pydvl python=3.8\nconda activate pydvl\npip install -r requirements-dev.txt -r requirements-docs.txt\n</code></pre> <p>A very convenient way of working with your library during development is to install it in editable mode into your environment by running</p> <pre><code>pip install -e .\n</code></pre> <p>In order to build the documentation locally (which is done as part of the tox suite) you need to install additional non-python dependencies as described in the documentation of mkdocs-material.</p> <p>In addition, pandoc is required. Except for OSX,  it should be installed automatically as a dependency with  <code>requirements-docs.txt</code>. Under OSX you can install pandoc  (you'll need at least version 2.11) with:</p> <pre><code>brew install pandoc\n</code></pre> <p>Remember to mark all autogenerated directories as excluded in your IDE. In particular <code>docs_build</code> and <code>.tox</code> should be marked as excluded to avoid slowdowns when searching or refactoring code.</p> <p>If you use remote execution, don't forget to exclude data paths from deployment (unless you really want to sync them).</p>"},{"location":"CONTRIBUTING/#testing","title":"Testing","text":"<p>Automated builds, tests, generation of documentation and publishing are handled by CI pipelines. Before pushing your changes to the remote we recommend to execute <code>tox</code> locally in order to detect mistakes early on and to avoid failing pipelines. tox will: * run the test suite * build the documentation * build and test installation of the package. * generate coverage and pylint reports in html, as well as badges.</p> <p>You can configure pytest, coverage and pylint by adjusting pyproject.toml.</p> <p>Besides the usual unit tests, most algorithms are tested using pytest. This requires ray for the parallelization and Memcached for caching. Please install both before running the tests. We run tests in CI as well.</p> <p>It is possible to pass optional command line arguments to pytest, for example to run only certain tests using patterns (<code>-k</code>) or marker (<code>-m</code>).</p> <pre><code>tox -e tests -- &lt;optional arguments&gt;\n</code></pre> <p>There are a few important arguments:</p> <ul> <li><code>--memcached-service</code> allows to change the default of <code>localhost:11211</code> (memcached's default)   to a different address.</li> </ul> <p>Memcached is needed for testing   caching as well as speeding certain methods (e.g. Permutation Shapley).</p> <p>To start memcached locally in the background with Docker use:</p> <pre><code>docker run --name pydvl-memcache -p 11211:11211 -d memcached\n</code></pre> <ul> <li><code>-n</code> sets the number of parallel workers for    pytest-xdist.</li> </ul> <p>There are two layers of parallelization in the tests.   An inner one within the tests themselves, i.e. the parallelism in the algorithms,   and an outer one by pytest-xdist. The latter is controlled by the <code>-n</code> argument.   If you experience segmentation faults with the tests,   try running them with <code>-n 0</code> to disable parallelization.</p> <ul> <li><code>--slow-tests</code> enables running slow tests. See below for a description   of slow tests.</li> </ul>"},{"location":"CONTRIBUTING/#markers","title":"Markers","text":"<p>We use a few different markers to differentiate between tests and runs groups of them of separately. Use <code>pytest --markers</code> to get a list and description of all available markers.</p> <p>Two important markers are:</p> <ul> <li><code>pytest.mark.slow</code> which is used to mark slow tests and skip them by default.</li> </ul> <p>A slow test is any test that takes 45 seconds or more to run and that can be   skipped most of the time. In some cases a test is slow, but it is required   in order to ensure that a feature works as expected and that are no bugs.   In those cases, we should not use this marker.</p> <p>Slow tests are always run on CI. Locally, they are skipped   by default but can be additionally run using: <code>pytest --slow-tests</code>.</p> <ul> <li><code>pytest.mark.torch</code> which is used to mark tests that require PyTorch.</li> </ul> <p>To test modules that rely on PyTorch, use:</p> <pre><code>tox -e tests -- -m \"torch\"\n</code></pre>"},{"location":"CONTRIBUTING/#other-things","title":"Other Things","text":"<p>To test the notebooks separately, run (see below for details):</p> <pre><code>tox -e notebook-tests\n</code></pre> <p>To create a package locally, run: <pre><code>python setup.py sdist bdist_wheel\n</code></pre></p>"},{"location":"CONTRIBUTING/#notebooks","title":"Notebooks","text":"<p>We use notebooks both as documentation (copied over to <code>docs/examples</code>) and as integration tests. All notebooks in the <code>notebooks</code> directory are executed during the test run. Because run times are typically too long for large datasets, you must check for the <code>CI</code> environment variable to work with smaller ones. For example, you can select a subset of the data:</p> <pre><code># In CI we only use a subset of the training set\nif os.environ.get('CI'):\n    training_data = training_data[:10]\n</code></pre> <p>This switching should happen in a separate notebook cell tagged with <code>hide</code> to hide the cell's input and output when rendering it as part of the documents. We want to avoid as much clutter and boilerplate as possible in the notebooks themselves.</p> <p>Because we want documentation to include the full dataset, we commit notebooks with their outputs running with full datasets to the repo. The notebooks are then added by CI to the section Examples of the documentation.</p>"},{"location":"CONTRIBUTING/#hiding-cells-in-notebooks","title":"Hiding cells in notebooks","text":"<p>Switching between CI or not, importing generic modules and plotting results are all examples of boilerplate code irrelevant to a reader interested in pyDVL's functionality. For this reason we choose to isolate this code into separate cells which are then hidden in the documentation.</p> <p>In order to do this, cells are marked with tags understood by the mkdocs plugin <code>mkdocs-jupyter</code>, namely adding the following to the metadata of the relevant cells:</p> <pre><code>\"tags\": [\n  \"hide\"\n]\n</code></pre> <p>To hide the cell's input and output.</p> <p>Or:</p> <pre><code>\"tags\": [\n  \"hide-input\"\n]\n</code></pre> <p>To only hide the input and</p> <p><pre><code>\"tags\": [\n  \"hide-output\"\n]\n</code></pre> for hiding the output only.</p> <p>It is important to leave a warning at the top of the document to avoid confusion. Examples for hidden imports and plots are available in the notebooks, e.g. in notebooks/shapley_basic_spotify.ipynb.</p>"},{"location":"CONTRIBUTING/#plots-in-notebooks","title":"Plots in Notebooks","text":"<p>If you add a plot to a notebook, which should also render nicely in browser dark mode, add the tag invertible-output, i.e.</p> <p><pre><code>\"tags\": [\n  \"invertible-output\"\n]\n</code></pre> This applies a simple CSS-filter to the output image of the cell.</p>"},{"location":"CONTRIBUTING/#documentation","title":"Documentation","text":"<p>API documentation and examples from notebooks are built with mkdocs, using a number of plugins, including mkdoctrings, with versioning handled by mike.</p> <p>Notebooks are an integral part of the documentation as well, please read the section on notebooks above.</p> <p>If you want to build the documentation locally, please make sure you followed the instructions in the section  Setting up your environment.</p> <p>Use the following command to build the documentation the same way it is done in CI:</p> <pre><code>mkdocs build\n</code></pre> <p>Locally, you can use this command instead to continuously rebuild documentation on changes to the <code>docs</code> and <code>src</code> folder:</p> <pre><code>mkdocs serve\n</code></pre> <p>This will rebuild the documentation on changes to <code>.md</code> files inside <code>docs</code>, notebooks and python files.</p> <p>On OSX, it is possible that the cairo lib file is not properly linked when installed via homebrew. In this case you might encounter an error like this <pre><code>OSError: no library called \"cairo-2\" was found\nno library called \"cairo\" was found\nno library called \"libcairo-2\" was found\n</code></pre> when calling <code>mkdocs build</code> or <code>mkdocs serve</code>. This can be resolved via setting the environment variable <code>DYLD_FALLBACK_LIBRARY_PATH</code>: <pre><code>export DYLD_FALLBACK_LIBRARY_PATH=$DYLD_FALLBACK_LIBRARY_PATH:/opt/homebrew/lib\n</code></pre></p>"},{"location":"CONTRIBUTING/#adding-new-pages","title":"Adding new pages","text":"<p>Navigation is configured in <code>mkdocs.yaml</code> using the nav section. We use the plugin mkdoc-literate-nav which allows fine-grained control of the navigation structure. However, most pages are explicitly listed and manually arranged in the <code>nav</code> section of the configuration.</p>"},{"location":"CONTRIBUTING/#creating-stable-references-for-autorefs","title":"Creating stable references for autorefs","text":"<p>mkdocstrings includes the plugin autorefs to enable automatic linking across pages with e.g. <code>[a link][to-something]</code>. Anchors are autogenerated from section titles, and are not guaranteed to be unique. In order to ensure that a link will remain valid, add a custom anchor to the section title:</p> <pre><code>## Some section { #permanent-anchor-to-some-section }\n</code></pre> <p>(note the space after the opening brace). You can then refer to it within another markdown file with <code>[Some section][permanent-anchor-to-some-section]</code>.</p>"},{"location":"CONTRIBUTING/#adding-notes-about-new-features-changes-or-deprecations","title":"Adding notes about new features, changes or deprecations","text":"<p>We use the admonition extension of Mkdocs Material to create admonitions, also known as call-outs, that hold information about when a certain feature was added, changed or deprecated and optionally a description with more details. We put the admonition directly in a module's, a function's or class' docstring.</p> <p>We use the following syntax:</p> <pre><code>!!! tip \"&lt;Event Type&gt; in version &lt;Version Number&gt;\"\n\n    &lt;Optional Description&gt;\n</code></pre> <p>The description is useful when the note is about a smaller change such as a parameter.</p> <ul> <li>For a new feature, we use:</li> </ul> <pre><code>!!! tip \"New in version &lt;Version Number&gt;\"\n\n    &lt;Optional Description&gt;\n</code></pre> <ul> <li>For a change to an existing feature we use:</li> </ul> <pre><code>!!! tip \"Changed in version &lt;Version Number&gt;\"\n\n    &lt;Optional Description&gt;\n</code></pre> <p>For example, for a change in version <code>1.2.3</code> that adds kwargs   to a class' constructor we would write:</p> <pre><code>!!! tip \"Changed in version 1.2.3\"\n\n    Added kwargs to the constructor.\n</code></pre> <ul> <li>For a deprecation we use:</li> </ul> <pre><code>!!! tip \"Deprecated in version &lt;Version Number&gt;\"\n\n    &lt;Optional Description&gt;\n</code></pre>"},{"location":"CONTRIBUTING/#using-bibliography","title":"Using bibliography","text":"<p>Bibliographic citations are managed with the plugin mkdocs-bibtex. To enter a citation first add the entry to <code>docs/pydvl.bib</code>. For team contributor this should be an export of the Zotero folder <code>software/pydvl</code> in the TransferLab Zotero library. All other contributors just add the bibtex data, and a maintainer will add it to the group library upon merging.</p> <p>To add a citation inside a markdown file, use the notation <code>[@citekey]</code>. Alas, because of when mkdocs-bibtex enters the pipeline, it won't process docstrings. For module documentation, we manually inject html into the markdown files. For example, in <code>pydvl.value.shapley.montecarlo</code> we have:</p> <pre><code>\"\"\"\nModule docstring...\n\n## References\n\n[^1]: &lt;a name=\"ghorbani_data_2019\"&gt;&lt;/a&gt;Ghorbani, A., Zou, J., 2019.\n    [Data Shapley: Equitable Valuation of Data for Machine\n    Learning](https://proceedings.mlr.press/v97/ghorbani19c.html).\n    In: Proceedings of the 36th International Conference on Machine Learning,\n    PMLR, pp. 2242\u20132251.\n\"\"\"\n</code></pre> <p>and then later in the file, inside a function's docstring:</p> <pre><code>    This function implements (Ghorbani and Zou, 2019)&lt;sup&gt;&lt;a \n    href=\"#ghorbani_data_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n</code></pre>"},{"location":"CONTRIBUTING/#writing-mathematics","title":"Writing mathematics","text":"<p>Use LaTeX delimiters <code>$</code> and <code>$$</code> for inline and displayed mathematics respectively.</p> <p>Warning: backslashes must be escaped in docstrings! (although there are exceptions). For simplicity, declare the string as \"raw\" with the prefix <code>r</code>:</p> <pre><code># This will work\ndef f(x: float) -&gt; float:\n    r\"\"\" Computes \n    $${ f(x) = \\frac{1}{x^2} }$$\n    \"\"\"\n    return 1/(x*x)\n\n# This throws an obscure error\ndef f(x: float) -&gt; float:\n    \"\"\" Computes \n    $$\\frac{1}{x^2}$$\n    \"\"\"\n    return 1/(x*x)\n</code></pre> <p>Note how there is no space after the dollar signs. This is important! You can use braces for legibility like in the first example.</p>"},{"location":"CONTRIBUTING/#abbreviations","title":"Abbreviations","text":"<p>We keep the abbreviations used in the documentation inside the docs_include/abbreviations.md file.</p> <p>The syntax for abbreviations is:</p> <pre><code>*[ABBR]: Abbreviation\n</code></pre>"},{"location":"CONTRIBUTING/#ci","title":"CI","text":"<p>We use workflows to:</p> <ul> <li>Run the tests.</li> <li>Publish documentation.</li> <li>Publish packages to testpypi / pypi.</li> <li>Mark issues as stale after 30 days. We do this only for issues with the label   <code>awaiting-reply</code>   which indicates that we have answered a question / feature request / PR and   are waiting for the OP to reply / update his work.</li> </ul>"},{"location":"CONTRIBUTING/#tests","title":"Tests","text":"<p>We test all algorithms with simple datasets in CI jobs. This can amount to a sizeable amount of time, so care must be taken not to overdo it: 1. All algorithm tests must be on very simple datasets and as quick as possible 2. We try not to trigger CI pipelines when unnecessary (see Skipping CI runs). 3. We split the tests based on their duration into groups and run them in parallel.</p> <p>For that we use pytest-split    to first store the duration of all tests with    <code>tox -e tests -- --store-durations --slow-tests</code>    in a <code>.test_durations</code> file.</p> <p>Alternatively, we case use pytest directly    <code>pytest --store-durations --slow-tests</code>.</p> <p>Note This does not have to be done each time a new test or test case is added. For new tests and test cases pytes-split assumes average test execution time(calculated based on the stored information) for every test which does not have duration information stored. Thus, there's no need to store durations after changing the test suite. However, when there are major changes in the suite compared to what's stored in .test_durations, it's recommended to update the duration information with <code>--store-durations</code> to ensure that the splitting is in balance.</p> <p>Then we can have as many splits as we want:</p> <pre><code>tox -e tests -- --splits 3 --group 1\ntox -e tests -- --splits 3 --group 2\ntox -e tests -- --splits 3 --group 3\n</code></pre> <p>Alternatively, we case use pytest directly    <code>pytest --splits 3 ---group 1</code>.</p> <p>Each one of these commands should be run in a separate shell/job    to run the test groups in parallel and decrease the total runtime.</p>"},{"location":"CONTRIBUTING/#running-github-actions-locally","title":"Running Github Actions locally","text":"<p>To run Github Actions locally we use act. It uses the workflows defined in <code>.github/workflows</code> and determines the set of actions that need to be run. It uses the Docker API to either pull or build the necessary images, as defined in our workflow files and finally determines the execution path based on the dependencies that were defined.</p> <p>Once it has the execution path, it then uses the Docker API to run containers for each action based on the images prepared earlier. The environment variables  and filesystem are all configured to match what GitHub provides.</p> <p>You can install it manually using:</p> <pre><code>curl -s https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash -s -- -d -b ~/bin \n</code></pre> <p>And then simply add it to your PATH variable: <code>PATH=~/bin:$PATH</code></p> <p>Refer to its official readme for more installation options.</p>"},{"location":"CONTRIBUTING/#act-cheatsheet","title":"act cheatsheet","text":"<p>By default, <code>act</code> will run all  workflows in <code>.github/workflows</code>. You can use the <code>-W</code> flag to specify a specific workflow file to run, or you can rely on the job id to be unique (but then you'll see warnings for the workflows without that job id).</p> <pre><code># Run only the main tests for python 3.8 after a push event (implicit) \nact -W .github/workflows/run-tests-workflow.yaml \\\n    -j run-tests \\\n    --input tests_to_run=base\\\n    --input python_version=3.8\n</code></pre> <p>Other common flags are: </p> <pre><code># List all actions for all events:\nact -l\n\n# List the actions for a specific event:\nact workflow_dispatch -l\n\n# List the actions for a specific job:\nact -j lint -l\n\n# Run the default (`push`) event:\nact\n\n# Run a specific event:\nact pull_request\n\n# Run a specific job:\nact -j lint\n\n# Collect artifacts to the /tmp/artifacts folder:\nact --artifact-server-path /tmp/artifacts\n\n# Run a job in a specific workflow (useful if you have duplicate job names)\nact -j lint -W .github/workflows/tox.yml\n\n# Run in dry-run mode:\nact -n\n\n# Enable verbose-logging (can be used with any of the above commands)\nact -v\n</code></pre>"},{"location":"CONTRIBUTING/#example","title":"Example","text":"<p>To run the <code>publish</code> job (the most difficult one to test) you would simply use:</p> <ul> <li>When triggered by a release:</li> </ul> <pre><code>act release -j publish --eventpath events.json\n</code></pre> <p>With <code>events.json</code> containing:</p> <pre><code>{\n  \"act\": true\n}\n</code></pre> <p>This will use your current branch. If you want to test a specific branch   you have to use the <code>workflow_dispatch</code> event (see below).</p> <ul> <li>To instead run it as if it had been manually triggered (i.e. <code>workflow_dispatch</code>)   you would instead use:</li> </ul> <pre><code>act workflow_dispatch -j publish --eventpath events.json\n</code></pre> <p>With <code>events.json</code> containing:</p> <pre><code>{\n  \"act\": true,\n  \"inputs\": {\n    \"tag_name\": \"v0.6.0\"\n  }\n}\n</code></pre>"},{"location":"CONTRIBUTING/#skipping-ci-runs","title":"Skipping CI runs","text":"<p>One sometimes would like to skip CI for certain commits (e.g. updating the readme). In order to do this, simply prefix the commit message with <code>[skip ci]</code>. The string can be anywhere, but adding it to the beginning of the commit message makes it more evident when looking at commits in a PR.</p> <p>Refer to the official GitHub documentation  for more information.</p>"},{"location":"CONTRIBUTING/#release-processes","title":"Release processes","text":""},{"location":"CONTRIBUTING/#automatic-release-process","title":"Automatic release process","text":"<p>In order to create an automatic release, a few prerequisites need to be satisfied:</p> <ul> <li>The project's virtualenv needs to be active</li> <li>The repository needs to be on the <code>develop</code> branch</li> <li>The repository must be clean (including no untracked files)</li> </ul> <p>Then, a new release can be created using the script <code>build_scripts/release-version.sh</code> (leave out the version parameter to have <code>bumpversion</code> automatically derive the next release version by bumping the patch part):</p> <pre><code>build_scripts/release-version.sh 0.1.6\n</code></pre> <p>To find out how to use the script, pass the <code>-h</code> or <code>--help</code> flags:</p> <pre><code>build_scripts/release-version.sh --help\n</code></pre> <p>If running in interactive mode (without <code>-y|--yes</code>), the script will output a summary of pending changes and ask for confirmation before executing the actions.</p> <p>Once this is done, a tag will be created on the repository. You should then create a GitHub release for that tag. That will a trigger a CI pipeline that will automatically create a package and publish it from CI to PyPI.</p>"},{"location":"CONTRIBUTING/#manual-release-process","title":"Manual release process","text":"<p>If the automatic release process doesn't cover your use case, you can also create a new release manually by following these steps:</p> <ol> <li>(Repeat as needed) implement features on feature branches merged into   <code>develop</code>. Each merge into develop will publish a new pre-release version     to TestPyPI. These versions can be installed using <code>pip install --pre     --index-url https://test.pypi.org/simple/</code>.</li> <li>When ready to release: From the develop branch create the release branch and    perform release activities (update changelog, news, ...). For your own    convenience, define an env variable for the release version     <pre><code>export RELEASE_VERSION=\"vX.Y.Z\"\ngit checkout develop\ngit branch release/${RELEASE_VERSION} &amp;&amp; git checkout release/${RELEASE_VERSION}\n</code></pre></li> <li>Run <code>bumpversion --commit release</code> if the release is only a patch release,    otherwise the full version can be specified using     <code>bumpversion --commit --new-version X.Y.Z release</code>    (the <code>release</code> part is ignored but required by bumpversion ).</li> <li>Merge the release branch into <code>master</code>, tag the merge commit, and push back to the repo.     The CI pipeline publishes the package based on the tagged commit.     <pre><code>git checkout master\ngit merge --no-ff release/${RELEASE_VERSION}\ngit tag -a ${RELEASE_VERSION} -m\"Release ${RELEASE_VERSION}\"\ngit push --follow-tags origin master\n</code></pre></li> <li>Switch back to the release branch <code>release/vX.Y.Z</code> and pre-bump the version:    <code>bumpversion --commit patch</code>. This ensures that <code>develop</code> pre-releases are    always strictly more recent than the last published release version from     <code>master</code>.</li> <li>Merge the release branch into <code>develop</code>:     <pre><code>git checkout develop\ngit merge --no-ff release/${RELEASE_VERSION}\ngit push origin develop\n</code></pre></li> <li>Delete the release branch if necessary:     <code>git branch -d release/${RELEASE_VERSION}</code></li> <li>Create a GitHub    release    for the created tag.</li> <li>Pour yourself a cup of coffee, you earned it!  </li> <li>A package will be automatically created and published from CI to PyPI.</li> </ol>"},{"location":"CONTRIBUTING/#ci-and-requirements-for-publishing","title":"CI and requirements for publishing","text":"<p>In order to publish new versions of the package from the development branch, the CI pipeline requires the following secret variables set up:</p> <pre><code>TEST_PYPI_USERNAME\nTEST_PYPI_PASSWORD\nPYPI_USERNAME\nPYPI_PASSWORD\n</code></pre> <p>The first 2 are used after tests run on the develop branch's CI workflow  to automatically publish packages to TestPyPI.</p> <p>The last 2 are used in the publish.yaml CI workflow to publish packages to PyPI from <code>develop</code> after a GitHub release.</p>"},{"location":"CONTRIBUTING/#publish-to-testpypi","title":"Publish to TestPyPI","text":"<p>We use bump2version to bump the build part of the version number without commiting or tagging the change and then publish a package to TestPyPI from CI using Twine. The version has the GitHub run number appended. </p> <p>For more details refer to the files .github/workflows/publish.yaml and .github/workflows/tox.yaml.</p>"},{"location":"api/pydvl/","title":"API Reference","text":""},{"location":"api/pydvl/#pydvl","title":"pydvl","text":""},{"location":"api/pydvl/#pydvl--the-python-data-valuation-library-api","title":"The Python Data Valuation Library API","text":"<p>This is the API documentation for the Python Data Valuation Library (PyDVL). Use the table of contents to access the documentation for each module.</p> <p>The two main modules you will want to look at are value and influence.</p>"},{"location":"api/pydvl/influence/","title":"Influence","text":""},{"location":"api/pydvl/influence/#pydvl.influence","title":"pydvl.influence","text":"<p>This package contains algorithms for the computation of the influence function.</p> <p>See The Influence function for an introduction to the concepts and methods implemented here.</p> <p>Warning</p> <p>Much of the code in this package is experimental or untested and is subject to modification. In particular, the package structure and basic API will probably change.</p>"},{"location":"api/pydvl/influence/array/","title":"Array","text":""},{"location":"api/pydvl/influence/array/#pydvl.influence.array","title":"pydvl.influence.array","text":"<p>This module provides classes and utilities for handling large arrays that are chunked and lazily evaluated. It includes abstract base classes for converting between tensor types and NumPy arrays, aggregating blocks of data, and abstract representations of lazy arrays. Concrete implementations are provided for handling chunked lazy arrays (chunked in one resp. two dimensions), with support for efficient storage and retrieval using the Zarr library.</p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NumpyConverter","title":"NumpyConverter","text":"<p>             Bases: <code>Generic[TensorType]</code>, <code>ABC</code></p> <p>Base class for converting TensorType objects into numpy arrays and vice versa.</p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NumpyConverter.to_numpy","title":"to_numpy  <code>abstractmethod</code>","text":"<pre><code>to_numpy(x: TensorType) -&gt; NDArray\n</code></pre> <p>Override this method for converting a TensorType object into a numpy array</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>@abstractmethod\ndef to_numpy(self, x: TensorType) -&gt; NDArray:\n    \"\"\"Override this method for converting a TensorType object into a numpy array\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NumpyConverter.from_numpy","title":"from_numpy  <code>abstractmethod</code>","text":"<pre><code>from_numpy(x: NDArray) -&gt; TensorType\n</code></pre> <p>Override this method for converting a numpy array into a TensorType object</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>@abstractmethod\ndef from_numpy(self, x: NDArray) -&gt; TensorType:\n    \"\"\"Override this method for converting a numpy array into a TensorType object\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.SequenceAggregator","title":"SequenceAggregator","text":"<p>             Bases: <code>Generic[TensorType]</code>, <code>ABC</code></p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.SequenceAggregator.__call__","title":"__call__  <code>abstractmethod</code>","text":"<pre><code>__call__(tensor_generator: Generator[TensorType, None, None])\n</code></pre> <p>Aggregates tensors from a generator.</p> <p>Implement this method to define how a sequence of tensors, provided by a generator, should be combined.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>@abstractmethod\ndef __call__(self, tensor_generator: Generator[TensorType, None, None]):\n    \"\"\"\n    Aggregates tensors from a generator.\n\n    Implement this method to define how a sequence of tensors, provided by a\n    generator, should be combined.\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.ListAggregator","title":"ListAggregator","text":"<p>             Bases: <code>SequenceAggregator</code></p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.ListAggregator.__call__","title":"__call__","text":"<pre><code>__call__(\n    tensor_generator: Generator[TensorType, None, None]\n) -&gt; List[TensorType]\n</code></pre> <p>Aggregates tensors from a single-level generator into a list. This method simply collects each tensor emitted by the generator into a single list.</p> PARAMETER  DESCRIPTION <code>tensor_generator</code> <p>A generator that yields TensorType objects.</p> <p> TYPE: <code>Generator[TensorType, None, None]</code> </p> RETURNS DESCRIPTION <code>List[TensorType]</code> <p>A list containing all the tensors provided by the tensor_generator.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def __call__(\n    self, tensor_generator: Generator[TensorType, None, None]\n) -&gt; List[TensorType]:\n    \"\"\"\n    Aggregates tensors from a single-level generator into a list. This method simply\n    collects each tensor emitted by the generator into a single list.\n\n    Args:\n        tensor_generator: A generator that yields TensorType objects.\n\n    Returns:\n        A list containing all the tensors provided by the tensor_generator.\n    \"\"\"\n    return [t for t in tensor_generator]\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedSequenceAggregator","title":"NestedSequenceAggregator","text":"<p>             Bases: <code>Generic[TensorType]</code>, <code>ABC</code></p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedSequenceAggregator.__call__","title":"__call__  <code>abstractmethod</code>","text":"<pre><code>__call__(\n    nested_generators_of_tensors: Generator[\n        Generator[TensorType, None, None], None, None\n    ]\n)\n</code></pre> <p>Aggregates tensors from a generator of generators.</p> <p>Implement this method to specify how tensors, nested in two layers of generators, should be combined. Useful for complex data structures where tensors are not directly accessible in a flat list.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>@abstractmethod\ndef __call__(\n    self,\n    nested_generators_of_tensors: Generator[\n        Generator[TensorType, None, None], None, None\n    ],\n):\n    \"\"\"\n    Aggregates tensors from a generator of generators.\n\n    Implement this method to specify how tensors, nested in two layers of\n    generators, should be combined. Useful for complex data structures where tensors\n    are not directly accessible in a flat list.\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedListAggregator","title":"NestedListAggregator","text":"<p>             Bases: <code>NestedSequenceAggregator</code></p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedListAggregator.__call__","title":"__call__","text":"<pre><code>__call__(\n    nested_generators_of_tensors: Generator[\n        Generator[TensorType, None, None], None, None\n    ]\n) -&gt; List[List[TensorType]]\n</code></pre> <p>Aggregates tensors from a nested generator structure into a list of lists.  Each inner generator is converted into a list of tensors, resulting in a nested  list structure.</p> <p>Args:      nested_generators_of_tensors: A generator of generators, where each inner         generator yields TensorType objects.</p> RETURNS DESCRIPTION <code>List[List[TensorType]]</code> <p>A list of lists, where each inner list contains tensors returned from one of the inner generators.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def __call__(\n    self,\n    nested_generators_of_tensors: Generator[\n        Generator[TensorType, None, None], None, None\n    ],\n) -&gt; List[List[TensorType]]:\n    \"\"\"\n     Aggregates tensors from a nested generator structure into a list of lists.\n     Each inner generator is converted into a list of tensors, resulting in a nested\n     list structure.\n\n     Args:\n         nested_generators_of_tensors: A generator of generators, where each inner\n            generator yields TensorType objects.\n\n    Returns:\n        A list of lists, where each inner list contains tensors returned from one\n            of the inner generators.\n    \"\"\"\n    return [list(tensor_gen) for tensor_gen in nested_generators_of_tensors]\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.LazyChunkSequence","title":"LazyChunkSequence","text":"<pre><code>LazyChunkSequence(\n    generator_factory: Callable[[], Generator[TensorType, None, None]]\n)\n</code></pre> <p>A class representing a chunked, and lazily evaluated array, where the chunking is restricted to the first dimension</p> <p>This class is designed to handle large arrays that don't fit in memory. It works by generating chunks of the array on demand and can also convert these chunks to a Zarr array for efficient storage and retrieval.</p> ATTRIBUTE DESCRIPTION <code>generator_factory</code> <p>A factory function that returns a generator. This generator yields chunks of the large array when called.</p> <p> </p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def __init__(\n    self, generator_factory: Callable[[], Generator[TensorType, None, None]]\n):\n    self.generator_factory = generator_factory\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.LazyChunkSequence.compute","title":"compute","text":"<pre><code>compute(aggregator: Optional[SequenceAggregator] = None)\n</code></pre> <p>Computes and optionally aggregates the chunks of the array using the provided aggregator. This method initiates the generation of chunks and then combines them according to the aggregator's logic.</p> PARAMETER  DESCRIPTION <code>aggregator</code> <p>An optional aggregator for combining the chunks of the array. If None, a default ListAggregator is used to simply collect the chunks into a list.</p> <p> TYPE: <code>Optional[SequenceAggregator]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <p>The aggregated result of all chunks of the array, the format of which depends on the aggregator used.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def compute(self, aggregator: Optional[SequenceAggregator] = None):\n    \"\"\"\n    Computes and optionally aggregates the chunks of the array using the provided\n    aggregator. This method initiates the generation of chunks and then\n    combines them according to the aggregator's logic.\n\n    Args:\n        aggregator: An optional aggregator for combining the chunks of\n            the array. If None, a default ListAggregator is used to simply collect\n            the chunks into a list.\n\n    Returns:\n        The aggregated result of all chunks of the array, the format of which\n            depends on the aggregator used.\n\n    \"\"\"\n    if aggregator is None:\n        aggregator = ListAggregator()\n    return aggregator(self.generator_factory())\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.LazyChunkSequence.to_zarr","title":"to_zarr","text":"<pre><code>to_zarr(\n    path_or_url: Union[str, StoreLike],\n    converter: NumpyConverter,\n    return_stored: bool = False,\n    overwrite: bool = False,\n) -&gt; Optional[Array]\n</code></pre> <p>Converts the array into Zarr format, a storage format optimized for large arrays, and stores it at the specified path or URL. This method is suitable for scenarios where the data needs to be saved for later use or for large datasets requiring efficient storage.</p> PARAMETER  DESCRIPTION <code>path_or_url</code> <p>The file path or URL where the Zarr array will be stored. Also excepts instances of zarr stores.</p> <p> TYPE: <code>Union[str, StoreLike]</code> </p> <code>converter</code> <p>A converter for transforming blocks into NumPy arrays compatible with Zarr.</p> <p> TYPE: <code>NumpyConverter</code> </p> <code>return_stored</code> <p>If True, the method returns the stored Zarr array; otherwise, it returns None.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>overwrite</code> <p>If True, overwrites existing data at the given path_or_url. If False, an error is raised in case of existing data.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Optional[Array]</code> <p>The Zarr array if return_stored is True; otherwise, None.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def to_zarr(\n    self,\n    path_or_url: Union[str, StoreLike],\n    converter: NumpyConverter,\n    return_stored: bool = False,\n    overwrite: bool = False,\n) -&gt; Optional[zarr.Array]:\n    \"\"\"\n    Converts the array into Zarr format, a storage format optimized for large\n    arrays, and stores it at the specified path or URL. This method is suitable for\n    scenarios where the data needs to be saved for later use or for large datasets\n    requiring efficient storage.\n\n    Args:\n        path_or_url: The file path or URL where the Zarr array will be stored.\n            Also excepts instances of zarr stores.\n        converter: A converter for transforming blocks into NumPy arrays\n            compatible with Zarr.\n        return_stored: If True, the method returns the stored Zarr array; otherwise,\n            it returns None.\n        overwrite: If True, overwrites existing data at the given path_or_url.\n            If False, an error is raised in case of existing data.\n\n    Returns:\n        The Zarr array if return_stored is True; otherwise, None.\n    \"\"\"\n    row_idx = 0\n    z = None\n    for block in self.generator_factory():\n        numpy_block = converter.to_numpy(block)\n\n        if z is None:\n            z = self._initialize_zarr_array(numpy_block, path_or_url, overwrite)\n\n        new_shape = self._new_shape_according_to_block(numpy_block, row_idx)\n        z.resize(new_shape)\n\n        z[row_idx : row_idx + numpy_block.shape[0]] = numpy_block\n        row_idx += numpy_block.shape[0]\n\n    return z if return_stored else None\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedLazyChunkSequence","title":"NestedLazyChunkSequence","text":"<pre><code>NestedLazyChunkSequence(\n    generator_factory: Callable[\n        [], Generator[Generator[TensorType, None, None], None, None]\n    ]\n)\n</code></pre> <p>A class representing chunked, and lazily evaluated array, where the chunking is restricted to the first two dimensions.</p> <p>This class is designed for handling large arrays where individual chunks are loaded and processed lazily. It supports converting these chunks into a Zarr array for efficient storage and retrieval, with chunking applied along the first two dimensions.</p> ATTRIBUTE DESCRIPTION <code>generator_factory</code> <p>A factory function that returns a generator of generators. Each inner generator yields chunks.</p> <p> </p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def __init__(\n    self,\n    generator_factory: Callable[\n        [], Generator[Generator[TensorType, None, None], None, None]\n    ],\n):\n    self.generator_factory = generator_factory\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedLazyChunkSequence.compute","title":"compute","text":"<pre><code>compute(aggregator: Optional[NestedSequenceAggregator] = None)\n</code></pre> <p>Computes and optionally aggregates the chunks of the array using the provided aggregator. This method initiates the generation of chunks and then combines them according to the aggregator's logic.</p> PARAMETER  DESCRIPTION <code>aggregator</code> <p>An optional aggregator for combining the chunks of the array. If None, a default NestedListAggregator is used to simply collect the chunks into a list of lists.</p> <p> TYPE: <code>Optional[NestedSequenceAggregator]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <p>The aggregated result of all chunks of the array, the format of which</p> <p>depends on the aggregator used.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def compute(self, aggregator: Optional[NestedSequenceAggregator] = None):\n    \"\"\"\n    Computes and optionally aggregates the chunks of the array using the provided\n    aggregator. This method initiates the generation of chunks and then\n    combines them according to the aggregator's logic.\n\n    Args:\n        aggregator: An optional aggregator for combining the chunks of\n            the array. If None, a default\n            [NestedListAggregator][pydvl.influence.array.NestedListAggregator]\n            is used to simply collect the chunks into a list of lists.\n\n    Returns:\n        The aggregated result of all chunks of the array, the format of which\n        depends on the aggregator used.\n\n    \"\"\"\n    if aggregator is None:\n        aggregator = NestedListAggregator()\n    return aggregator(self.generator_factory())\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedLazyChunkSequence.to_zarr","title":"to_zarr","text":"<pre><code>to_zarr(\n    path_or_url: Union[str, StoreLike],\n    converter: NumpyConverter,\n    return_stored: bool = False,\n    overwrite: bool = False,\n) -&gt; Optional[Array]\n</code></pre> <p>Converts the array into Zarr format, a storage format optimized for large arrays, and stores it at the specified path or URL. This method is suitable for scenarios where the data needs to be saved for later use or for large datasets requiring efficient storage.</p> PARAMETER  DESCRIPTION <code>path_or_url</code> <p>The file path or URL where the Zarr array will be stored. Also excepts instances of zarr stores.</p> <p> TYPE: <code>Union[str, StoreLike]</code> </p> <code>converter</code> <p>A converter for transforming blocks into NumPy arrays compatible with Zarr.</p> <p> TYPE: <code>NumpyConverter</code> </p> <code>return_stored</code> <p>If True, the method returns the stored Zarr array; otherwise, it returns None.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>overwrite</code> <p>If True, overwrites existing data at the given path_or_url. If False, an error is raised in case of existing data.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Optional[Array]</code> <p>The Zarr array if return_stored is True; otherwise, None.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def to_zarr(\n    self,\n    path_or_url: Union[str, StoreLike],\n    converter: NumpyConverter,\n    return_stored: bool = False,\n    overwrite: bool = False,\n) -&gt; Optional[zarr.Array]:\n    \"\"\"\n    Converts the array into Zarr format, a storage format optimized for large\n    arrays, and stores it at the specified path or URL. This method is suitable for\n    scenarios where the data needs to be saved for later use or for large datasets\n    requiring efficient storage.\n\n    Args:\n        path_or_url: The file path or URL where the Zarr array will be stored.\n            Also excepts instances of zarr stores.\n        converter: A converter for transforming blocks into NumPy arrays\n            compatible with Zarr.\n        return_stored: If True, the method returns the stored Zarr array;\n            otherwise, it returns None.\n        overwrite: If True, overwrites existing data at the given path_or_url.\n            If False, an error is raised in case of existing data.\n\n    Returns:\n        The Zarr array if return_stored is True; otherwise, None.\n    \"\"\"\n\n    row_idx = 0\n    z = None\n    numpy_block = None\n    for row_blocks in self.generator_factory():\n        col_idx = 0\n        for block in row_blocks:\n            numpy_block = converter.to_numpy(block)\n            if z is None:\n                z = self._initialize_zarr_array(numpy_block, path_or_url, overwrite)\n            new_shape = self._new_shape_according_to_block(\n                z, numpy_block, row_idx, col_idx\n            )\n            z.resize(new_shape)\n            idx_slice_to_update = self._idx_slice_for_update(\n                numpy_block, row_idx, col_idx\n            )\n            z[idx_slice_to_update] = numpy_block\n\n            col_idx += numpy_block.shape[1]\n\n        if numpy_block is None:\n            raise ValueError(\"Generator is empty\")\n\n        row_idx += numpy_block.shape[0]\n\n    return z if return_stored else None\n</code></pre>"},{"location":"api/pydvl/influence/base_influence_function_model/","title":"Base influence function model","text":""},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model","title":"pydvl.influence.base_influence_function_model","text":""},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceMode","title":"InfluenceMode","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Enum representation for the types of influence.</p> ATTRIBUTE DESCRIPTION <code>Up</code> <p>Approximating the influence of a point</p> <p> </p> <code>Perturbation</code> <p>Perturbation definition of the influence score</p> <p> </p>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel","title":"InfluenceFunctionModel","text":"<p>             Bases: <code>Generic[TensorType, DataLoaderType]</code>, <code>ABC</code></p> <p>Generic abstract base class for computing influence related quantities. For a specific influence algorithm and tensor framework, inherit from this base class</p>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel.n_parameters","title":"n_parameters  <code>abstractmethod</code> <code>property</code>","text":"<pre><code>n_parameters\n</code></pre> <p>Number of trainable parameters of the underlying model</p>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel.is_thread_safe","title":"is_thread_safe  <code>abstractmethod</code> <code>property</code>","text":"<pre><code>is_thread_safe: bool\n</code></pre> <p>Whether the influence computation is thread safe</p>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel.is_fitted","title":"is_fitted  <code>abstractmethod</code> <code>property</code>","text":"<pre><code>is_fitted\n</code></pre> <p>Override this, to expose the fitting status of the instance.</p>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel.fit","title":"fit  <code>abstractmethod</code>","text":"<pre><code>fit(data: DataLoaderType) -&gt; InfluenceFunctionModel\n</code></pre> <p>Override this method to fit the influence function model to training data, e.g. pre-compute hessian matrix or matrix decompositions</p> PARAMETER  DESCRIPTION <code>data</code> <p> TYPE: <code>DataLoaderType</code> </p> RETURNS DESCRIPTION <code>InfluenceFunctionModel</code> <p>The fitted instance</p> Source code in <code>src/pydvl/influence/base_influence_function_model.py</code> <pre><code>@abstractmethod\ndef fit(self, data: DataLoaderType) -&gt; InfluenceFunctionModel:\n    \"\"\"\n    Override this method to fit the influence function model to training data,\n    e.g. pre-compute hessian matrix or matrix decompositions\n\n    Args:\n        data:\n\n    Returns:\n        The fitted instance\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel.influences_from_factors","title":"influences_from_factors  <code>abstractmethod</code>","text":"<pre><code>influences_from_factors(\n    z_test_factors: TensorType,\n    x: TensorType,\n    y: TensorType,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; TensorType\n</code></pre> <p>Override this method to implement the computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed array, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>TensorType</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>TensorType</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>TensorType</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/base_influence_function_model.py</code> <pre><code>@abstractmethod\ndef influences_from_factors(\n    self,\n    z_test_factors: TensorType,\n    x: TensorType,\n    y: TensorType,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; TensorType:\n    r\"\"\"\n    Override this method to implement the computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$.\n\n    Args:\n        z_test_factors: pre-computed array, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/","title":"Influence calculator","text":""},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator","title":"pydvl.influence.influence_calculator","text":"<p>This module provides functionality for calculating influences for large amount of data. The computation is based on a chunk computation model in the form of an instance of InfluenceFunctionModel, which is mapped over collection of chunks.</p>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DisableClientSingleThreadCheck","title":"DisableClientSingleThreadCheck","text":"<p>This type can be provided to the initialization of a DaskInfluenceCalculator instead of a distributed client object. It is useful in those scenarios, where the user want to disable the checking for thread-safety in the initialization phase, e.g. when using the single machine synchronous scheduler for debugging purposes.</p> Example <pre><code>from pydvl.influence import DisableClientThreadingCheck\n\nda_calc = DaskInfluenceCalculator(if_model,\n                                  TorchNumpyConverter(),\n                                  DisableClientThreadingCheck)\nda_influences = da_calc.influences(da_x_test, da_y_test, da_x, da_y)\nda_influences.compute(scheduler='synchronous')\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DaskInfluenceCalculator","title":"DaskInfluenceCalculator","text":"<pre><code>DaskInfluenceCalculator(\n    influence_function_model: InfluenceFunctionModel,\n    converter: NumpyConverter,\n    client: Union[Client, Type[DisableClientSingleThreadCheck]],\n)\n</code></pre> <p>This class is designed to compute influences over dask.array.Array collections, leveraging the capabilities of Dask for distributed computing and parallel processing. It requires an influence computation model of type InfluenceFunctionModel, which defines how influences are computed on a chunk of data. Essentially, this class functions by mapping the influence function model across the various chunks of a dask.array.Array collection.</p> PARAMETER  DESCRIPTION <code>influence_function_model</code> <p>instance of type InfluenceFunctionModel, that specifies the computation logic for influence on data chunks. It's a pivotal part of the calculator, determining how influence is computed and applied across the data array.</p> <p> TYPE: <code>InfluenceFunctionModel</code> </p> <code>converter</code> <p>A utility for converting numpy arrays to TensorType objects, facilitating the interaction between numpy arrays and the influence function model.</p> <p> TYPE: <code>NumpyConverter</code> </p> <code>client</code> <p>This parameter accepts either of two types:</p> <ol> <li> <p>A distributed Client object</p> </li> <li> <p>The special type DisableClientSingleThreadCheck, which serves as a flag to bypass certain checks.</p> </li> </ol> <p>During initialization, the system verifies if all workers are operating in single-threaded mode when the provided influence_function_model is designated as not thread-safe (indicated by the <code>is_thread_safe</code> property). If this condition is not met, the initialization will raise a specific error, signaling a potential thread-safety conflict.</p> <p>To intentionally skip this safety check (e.g., for debugging purposes using the single machine synchronous scheduler), you can supply the DisableClientSingleThreadCheck type.</p> <p> TYPE: <code>Union[Client, Type[DisableClientSingleThreadCheck]]</code> </p> <p>Warning</p> <p>Make sure to set <code>threads_per_worker=1</code>, when using the distributed scheduler for computing, if your implementation of InfluenceFunctionModel is not thread-safe. <pre><code>client = Client(threads_per_worker=1)\n</code></pre> For details on dask schedulers see the official documentation.</p> Example <pre><code>import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom pydvl.influence import DaskInfluenceCalculator\nfrom pydvl.influence.torch import CgInfluence\nfrom pydvl.influence.torch.util import (\n    torch_dataset_to_dask_array,\n    TorchNumpyConverter,\n)\nfrom distributed import Client\n\n# Possible some out of memory large Dataset\ntrain_data_set: Dataset = LargeDataSet(...)\ntest_data_set: Dataset = LargeDataSet(...)\n\ntrain_dataloader = DataLoader(train_data_set)\ninfl_model = CgInfluence(model, loss, hessian_regularization=0.01)\ninfl_model = if_model.fit(train_dataloader)\n\n# wrap your input data into dask arrays\nchunk_size = 10\nda_x, da_y = torch_dataset_to_dask_array(train_data_set, chunk_size=chunk_size)\nda_x_test, da_y_test = torch_dataset_to_dask_array(test_data_set,\n                                                   chunk_size=chunk_size)\n\n# use only one thread for scheduling, due to non-thread safety of some torch\n# operations\nclient = Client(n_workers=4, threads_per_worker=1)\n\ninfl_calc = DaskInfluenceCalculator(infl_model,\n                                    TorchNumpyConverter(device=torch.device(\"cpu\")),\n                                    client)\nda_influences = infl_calc.influences(da_x_test, da_y_test, da_x, da_y)\n# da_influences is a dask.array.Array\n\n# trigger computation and write chunks to disk in parallel\nda_influences.to_zarr(\"path/or/url\")\n</code></pre> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def __init__(\n    self,\n    influence_function_model: InfluenceFunctionModel,\n    converter: NumpyConverter,\n    client: Union[Client, Type[DisableClientSingleThreadCheck]],\n):\n    self._n_parameters = influence_function_model.n_parameters\n    self.influence_function_model = influence_function_model\n    self.numpy_converter = converter\n\n    if isinstance(client, type(DisableClientSingleThreadCheck)):\n        logger.warning(DisableClientSingleThreadCheck.warning_msg())\n        self.influence_function_model = delayed(influence_function_model)\n    elif isinstance(client, Client):\n        self._validate_client(client, influence_function_model)\n        self.influence_function_model = client.scatter(\n            influence_function_model, broadcast=True\n        )\n    else:\n        raise ValueError(\n            \"The 'client' parameter \"\n            \"must either be a distributed.Client object or the\"\n            \"type 'DisableClientSingleThreadCheck'.\"\n        )\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DaskInfluenceCalculator.n_parameters","title":"n_parameters  <code>property</code>","text":"<pre><code>n_parameters\n</code></pre> <p>Number of trainable parameters of the underlying model used in the batch computation</p>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DaskInfluenceCalculator.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Array, y: Array) -&gt; Array\n</code></pre> <p>Computes the expression</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradients are computed for the chunks of \\((x, y)\\).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Array</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Array</code> </p> RETURNS DESCRIPTION <code>Array</code> <p>dask.array.Array representing the element-wise inverse Hessian matrix vector products for the provided batch.</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influence_factors(self, x: da.Array, y: da.Array) -&gt; da.Array:\n    r\"\"\"\n    Computes the expression\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradients are computed for the chunks of $(x, y)$.\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        [dask.array.Array][dask.array.Array] representing the element-wise inverse\n            Hessian matrix vector products for the provided batch.\n\n    \"\"\"\n\n    self._validate_aligned_chunking(x, y)\n    self._validate_dimensions_not_chunked(x)\n    self._validate_dimensions_not_chunked(y)\n\n    def func(x_numpy: NDArray, y_numpy: NDArray, model: InfluenceFunctionModel):\n        factors = model.influence_factors(\n            self.numpy_converter.from_numpy(x_numpy),\n            self.numpy_converter.from_numpy(y_numpy),\n        )\n        return self.numpy_converter.to_numpy(factors)\n\n    chunks = []\n    for x_chunk, y_chunk, chunk_size in zip(\n        x.to_delayed(), y.to_delayed(), x.chunks[0]\n    ):\n        chunk_shape = (chunk_size, self.n_parameters)\n        chunk_array = da.from_delayed(\n            delayed(func)(\n                x_chunk.squeeze()[()],\n                y_chunk.squeeze()[()],\n                self.influence_function_model,\n            ),\n            dtype=x.dtype,\n            shape=chunk_shape,\n        )\n        chunks.append(chunk_array)\n\n    return da.concatenate(chunks)\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DaskInfluenceCalculator.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Array,\n    y_test: Array,\n    x: Optional[Array] = None,\n    y: Optional[Array] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Array\n</code></pre> <p>Compute approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The computation is done block-wise for the chunks of the provided dask arrays.</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Array</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Array</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Array]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Array]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Array</code> <p>dask.array.Array representing the element-wise scalar products for the provided batch.</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influences(\n    self,\n    x_test: da.Array,\n    y_test: da.Array,\n    x: Optional[da.Array] = None,\n    y: Optional[da.Array] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; da.Array:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})),\n    \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The computation is done block-wise\n    for the chunks of the provided dask arrays.\n\n    Args:\n        x_test: model input to use in the gradient computations of\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        [dask.array.Array][dask.array.Array] representing the element-wise scalar\n            products for the provided batch.\n\n    \"\"\"\n\n    self._validate_aligned_chunking(x_test, y_test)\n    self._validate_dimensions_not_chunked(x_test)\n    self._validate_dimensions_not_chunked(y_test)\n\n    if (x is None) != (y is None):\n        if x is None:\n            raise ValueError(\n                \"Providing labels y without providing model input x \"\n                \"is not supported\"\n            )\n        if y is None:\n            raise ValueError(\n                \"Providing model input x without labels y is not supported\"\n            )\n    elif x is not None:\n        self._validate_aligned_chunking(x, y)\n        self._validate_dimensions_not_chunked(x)\n        self._validate_dimensions_not_chunked(y)\n    else:\n        x, y = x_test, y_test\n\n    def func(\n        x_test_numpy: NDArray,\n        y_test_numpy: NDArray,\n        x_numpy: NDArray,\n        y_numpy: NDArray,\n        model: InfluenceFunctionModel,\n    ):\n        values = model.influences(\n            self.numpy_converter.from_numpy(x_test_numpy),\n            self.numpy_converter.from_numpy(y_test_numpy),\n            self.numpy_converter.from_numpy(x_numpy),\n            self.numpy_converter.from_numpy(y_numpy),\n            mode,\n        )\n        return self.numpy_converter.to_numpy(values)\n\n    un_chunked_x_shapes = [s[0] for s in x_test.chunks[1:]]\n    x_test_chunk_sizes = x_test.chunks[0]\n    x_chunk_sizes = x.chunks[0]\n    blocks = []\n    block_shape: Tuple[int, ...]\n\n    for x_test_chunk, y_test_chunk, test_chunk_size in zip(\n        x_test.to_delayed(), y_test.to_delayed(), x_test_chunk_sizes\n    ):\n        row = []\n        for x_chunk, y_chunk, chunk_size in zip(\n            x.to_delayed(), y.to_delayed(), x_chunk_sizes  # type:ignore\n        ):\n            if mode == InfluenceMode.Up:\n                block_shape = (test_chunk_size, chunk_size)\n            elif mode == InfluenceMode.Perturbation:\n                block_shape = (test_chunk_size, chunk_size, *un_chunked_x_shapes)\n            else:\n                raise UnsupportedInfluenceModeException(mode)\n\n            block_array = da.from_delayed(\n                delayed(func)(\n                    x_test_chunk.squeeze()[()],\n                    y_test_chunk.squeeze()[()],\n                    x_chunk.squeeze()[()],\n                    y_chunk.squeeze()[()],\n                    self.influence_function_model,\n                ),\n                shape=block_shape,\n                dtype=x_test.dtype,\n            )\n\n            if mode == InfluenceMode.Perturbation:\n                n_dims = block_array.ndim\n                new_order = tuple(range(2, n_dims)) + (0, 1)\n                block_array = block_array.transpose(new_order)\n\n            row.append(block_array)\n        blocks.append(row)\n\n    values_array = da.block(blocks)\n\n    if mode == InfluenceMode.Perturbation:\n        n_dims = values_array.ndim\n        new_order = (n_dims - 2, n_dims - 1) + tuple(range(n_dims - 2))\n        values_array = values_array.transpose(new_order)\n\n    return values_array\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DaskInfluenceCalculator.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Array,\n    x: Array,\n    y: Array,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Array\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed array, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Array</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Array</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Array</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Array</code> <p>dask.array.Array representing the element-wise scalar product of the provided batch</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: da.Array,\n    x: da.Array,\n    y: da.Array,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; da.Array:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant\n    to be per sample of the batch $(x, y)$.\n\n    Args:\n        z_test_factors: pre-computed array, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n      [dask.array.Array][dask.array.Array] representing the element-wise scalar\n        product of the provided batch\n\n    \"\"\"\n    self._validate_aligned_chunking(x, y)\n    self._validate_dimensions_not_chunked(x)\n    self._validate_dimensions_not_chunked(y)\n    self._validate_dimensions_not_chunked(z_test_factors)\n\n    def func(\n        z_test_numpy: NDArray,\n        x_numpy: NDArray,\n        y_numpy: NDArray,\n        model: InfluenceFunctionModel,\n    ):\n        ups = model.influences_from_factors(\n            self.numpy_converter.from_numpy(z_test_numpy),\n            self.numpy_converter.from_numpy(x_numpy),\n            self.numpy_converter.from_numpy(y_numpy),\n            mode=mode,\n        )\n        return self.numpy_converter.to_numpy(ups)\n\n    un_chunked_x_shape = [s[0] for s in x.chunks[1:]]\n    x_chunk_sizes = x.chunks[0]\n    z_test_chunk_sizes = z_test_factors.chunks[0]\n    blocks = []\n    block_shape: Tuple[int, ...]\n\n    for z_test_chunk, z_test_chunk_size in zip(\n        z_test_factors.to_delayed(), z_test_chunk_sizes\n    ):\n        row = []\n        for x_chunk, y_chunk, chunk_size in zip(\n            x.to_delayed(), y.to_delayed(), x_chunk_sizes\n        ):\n            if mode == InfluenceMode.Perturbation:\n                block_shape = (z_test_chunk_size, chunk_size, *un_chunked_x_shape)\n            elif mode == InfluenceMode.Up:\n                block_shape = (z_test_chunk_size, chunk_size)\n            else:\n                raise UnsupportedInfluenceModeException(mode)\n\n            block_array = da.from_delayed(\n                delayed(func)(\n                    z_test_chunk.squeeze()[()],\n                    x_chunk.squeeze()[()],\n                    y_chunk.squeeze()[()],\n                    self.influence_function_model,\n                ),\n                shape=block_shape,\n                dtype=z_test_factors.dtype,\n            )\n\n            if mode == InfluenceMode.Perturbation:\n                n_dims = block_array.ndim\n                new_order = tuple(range(2, n_dims)) + (0, 1)\n                block_array = block_array.transpose(*new_order)\n\n            row.append(block_array)\n        blocks.append(row)\n\n    values_array = da.block(blocks)\n\n    if mode == InfluenceMode.Perturbation:\n        n_dims = values_array.ndim\n        new_order = (n_dims - 2, n_dims - 1) + tuple(range(n_dims - 2))\n        values_array = values_array.transpose(*new_order)\n\n    return values_array\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.SequentialInfluenceCalculator","title":"SequentialInfluenceCalculator","text":"<pre><code>SequentialInfluenceCalculator(influence_function_model: InfluenceFunctionModel)\n</code></pre> <p>This class serves as a simple wrapper for processing batches of data in a sequential manner. It is particularly useful in scenarios where parallel or distributed processing is not required or not feasible. The core functionality of this class is to apply a specified influence computation model, of type InfluenceFunctionModel, to batches of data one at a time.</p> PARAMETER  DESCRIPTION <code>influence_function_model</code> <p>An instance of type     [InfluenceFunctionModel]     [pydvl.influence.base_influence_function_model.InfluenceFunctionModel], that     specifies the computation logic for influence on data chunks.</p> <p> TYPE: <code>InfluenceFunctionModel</code> </p> Example <pre><code>from pydvl.influence import SequentialInfluenceCalculator\nfrom pydvl.influence.torch.util import (\nNestedTorchCatAggregator,\nTorchNumpyConverter,\n)\nfrom pydvl.influence.torch import CgInfluence\n\nbatch_size = 10\ntrain_dataloader = DataLoader(..., batch_size=batch_size)\ntest_dataloader = DataLoader(..., batch_size=batch_size)\n\ninfl_model = CgInfluence(model, loss, hessian_regularization=0.01)\ninfl_model = infl_model.fit(train_dataloader)\n\ninfl_calc = SequentialInfluenceCalculator(if_model)\n\n# this does not trigger the computation\nlazy_influences = infl_calc.influences(test_dataloader, train_dataloader)\n\n# trigger computation and pull the result into main memory, result is the full\n# tensor for all combinations of the two loaders\ninfluences = lazy_influences.compute(aggregator=NestedTorchCatAggregator())\n# or\n# trigger computation and write results chunk-wise to disk using zarr in a\n# sequential manner\nlazy_influences.to_zarr(\"local_path/or/url\", TorchNumpyConverter())\n</code></pre> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def __init__(\n    self,\n    influence_function_model: InfluenceFunctionModel,\n):\n    self.influence_function_model = influence_function_model\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.SequentialInfluenceCalculator.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(\n    data_iterable: Iterable[Tuple[TensorType, TensorType]]\n) -&gt; LazyChunkSequence\n</code></pre> <p>Compute the expression</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient are computed for the chunks \\((x, y)\\) of the data_iterable in a sequential manner.</p> PARAMETER  DESCRIPTION <code>data_iterable</code> <p>An iterable that returns tuples of tensors. Each tuple consists of a pair of tensors (x, y), representing input data and corresponding targets.</p> <p> TYPE: <code>Iterable[Tuple[TensorType, TensorType]]</code> </p> RETURNS DESCRIPTION <code>LazyChunkSequence</code> <p>A lazy data structure representing the chunks of the resulting tensor</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influence_factors(\n    self,\n    data_iterable: Iterable[Tuple[TensorType, TensorType]],\n) -&gt; LazyChunkSequence:\n    r\"\"\"\n    Compute the expression\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient are computed for the chunks $(x, y)$ of the data_iterable in\n    a sequential manner.\n\n    Args:\n        data_iterable: An iterable that returns tuples of tensors.\n            Each tuple consists of a pair of tensors (x, y), representing input data\n            and corresponding targets.\n\n    Returns:\n        A lazy data structure representing the chunks of the resulting tensor\n    \"\"\"\n    tensors_gen_factory = partial(self._influence_factors_gen, data_iterable)\n    return LazyChunkSequence(tensors_gen_factory)\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.SequentialInfluenceCalculator.influences","title":"influences","text":"<pre><code>influences(\n    test_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    train_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; NestedLazyChunkSequence\n</code></pre> <p>Compute approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The computation is done block-wise for the chunks of the provided data iterables and aggregated into a single tensor in memory.</p> PARAMETER  DESCRIPTION <code>test_data_iterable</code> <p>An iterable that returns tuples of tensors. Each tuple consists of a pair of tensors (x, y), representing input data and corresponding targets.</p> <p> TYPE: <code>Iterable[Tuple[TensorType, TensorType]]</code> </p> <code>train_data_iterable</code> <p>An iterable that returns tuples of tensors. Each tuple consists of a pair of tensors (x, y), representing input data and corresponding targets.</p> <p> TYPE: <code>Iterable[Tuple[TensorType, TensorType]]</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>NestedLazyChunkSequence</code> <p>A lazy data structure representing the chunks of the resulting tensor</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influences(\n    self,\n    test_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    train_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; NestedLazyChunkSequence:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})),\n    \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The computation is done block-wise for\n    the chunks of the provided\n    data iterables and aggregated into a single tensor in memory.\n\n    Args:\n        test_data_iterable: An iterable that returns tuples of tensors.\n            Each tuple consists of a pair of tensors (x, y), representing input data\n            and corresponding targets.\n        train_data_iterable: An iterable that returns tuples of tensors.\n            Each tuple consists of a pair of tensors (x, y), representing input data\n            and corresponding targets.\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        A lazy data structure representing the chunks of the resulting tensor\n\n    \"\"\"\n    nested_tensor_gen_factory = partial(\n        self._influences_gen,\n        test_data_iterable,\n        train_data_iterable,\n        mode,\n    )\n\n    return NestedLazyChunkSequence(nested_tensor_gen_factory)\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.SequentialInfluenceCalculator.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Iterable[TensorType],\n    train_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; NestedLazyChunkSequence\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}}, \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))     \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}}, \\nabla_{x} \\nabla_{\\theta}     \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>Pre-computed iterable of tensors, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Iterable[TensorType]</code> </p> <code>train_data_iterable</code> <p>An iterable that returns tuples of tensors. Each tuple consists of a pair of tensors (x, y), representing input data and corresponding targets.</p> <p> TYPE: <code>Iterable[Tuple[TensorType, TensorType]]</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>NestedLazyChunkSequence</code> <p>A lazy data structure representing the chunks of the resulting tensor</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: Iterable[TensorType],\n    train_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; NestedLazyChunkSequence:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}}, \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\n        \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}}, \\nabla_{x} \\nabla_{\\theta}\n        \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$.\n\n    Args:\n        z_test_factors: Pre-computed iterable of tensors, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        train_data_iterable: An iterable that returns tuples of tensors.\n            Each tuple consists of a pair of tensors (x, y), representing input data\n            and corresponding targets.\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n      A lazy data structure representing the chunks of the resulting tensor\n\n    \"\"\"\n    nested_tensor_gen = partial(\n        self._influences_from_factors_gen,\n        z_test_factors,\n        train_data_iterable,\n        mode,\n    )\n    return NestedLazyChunkSequence(nested_tensor_gen)\n</code></pre>"},{"location":"api/pydvl/influence/torch/","title":"Torch","text":""},{"location":"api/pydvl/influence/torch/#pydvl.influence.torch","title":"pydvl.influence.torch","text":""},{"location":"api/pydvl/influence/torch/functional/","title":"Functional","text":""},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional","title":"pydvl.influence.torch.functional","text":"<p>This module provides methods for efficiently computing tensors related to first and second order derivatives of torch models, using functionality from torch.func. To indicate higher-order functions, i.e. functions which return functions, we use the naming convention <code>create_**_function</code>.</p> <p>In particular, the module contains functionality for</p> <ul> <li>Sample, batch-wise and empirical loss functions:<ul> <li>create_per_sample_loss_function</li> <li>create_batch_loss_function</li> <li>create_empirical_loss_function</li> </ul> </li> <li>Per sample gradient and jacobian product functions:<ul> <li>create_per_sample_gradient_function</li> <li>create_per_sample_mixed_derivative_function</li> <li>create_matrix_jacobian_product_function</li> </ul> </li> <li>Hessian, low rank approximation of Hessian and Hessian vector products:<ul> <li>hvp</li> <li>create_hvp_function</li> <li>create_batch_hvp_function</li> <li>hessian</li> <li>model_hessian_low_rank</li> </ul> </li> </ul>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.LowRankProductRepresentation","title":"LowRankProductRepresentation  <code>dataclass</code>","text":"<pre><code>LowRankProductRepresentation(eigen_vals: Tensor, projections: Tensor)\n</code></pre> <p>Representation of a low rank product of the form \\(H = V D V^T\\), where D is a diagonal matrix and V is orthogonal.</p> PARAMETER  DESCRIPTION <code>eigen_vals</code> <p>Diagonal of D.</p> <p> TYPE: <code>Tensor</code> </p> <code>projections</code> <p>The matrix V.</p> <p> TYPE: <code>Tensor</code> </p>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.LowRankProductRepresentation.to","title":"to","text":"<pre><code>to(device: device)\n</code></pre> <p>Move the representing tensors to a device</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def to(self, device: torch.device):\n    \"\"\"\n    Move the representing tensors to a device\n    \"\"\"\n    return LowRankProductRepresentation(\n        self.eigen_vals.to(device), self.projections.to(device)\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.hvp","title":"hvp","text":"<pre><code>hvp(\n    func: Callable[[Dict[str, Tensor]], Tensor],\n    params: Dict[str, Tensor],\n    vec: Dict[str, Tensor],\n    reverse_only: bool = True,\n) -&gt; Dict[str, Tensor]\n</code></pre> <p>Computes the Hessian-vector product (HVP) for a given function at the given parameters, i.e.</p> \\[\\nabla_{\\theta} \\nabla_{\\theta} f (\\theta)\\cdot v\\] <p>This function can operate in two modes, either reverse-mode autodiff only or both forward- and reverse-mode autodiff.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The scalar-valued function for which the HVP is computed.</p> <p> TYPE: <code>Callable[[Dict[str, Tensor]], Tensor]</code> </p> <code>params</code> <p>The parameters at which the HVP is computed.</p> <p> TYPE: <code>Dict[str, Tensor]</code> </p> <code>vec</code> <p>The vector with which the Hessian is multiplied.</p> <p> TYPE: <code>Dict[str, Tensor]</code> </p> <code>reverse_only</code> <p>Whether to use only reverse-mode autodiff (True, default) or both forward- and reverse-mode autodiff (False).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>The HVP of the function at the given parameters with the given vector.</p> Example <pre><code>&gt;&gt;&gt; def f(z): return torch.sum(z**2)\n&gt;&gt;&gt; u = torch.ones(10, requires_grad=True)\n&gt;&gt;&gt; v = torch.ones(10)\n&gt;&gt;&gt; hvp_vec = hvp(f, u, v)\n&gt;&gt;&gt; assert torch.allclose(hvp_vec, torch.full((10, ), 2.0))\n</code></pre> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def hvp(\n    func: Callable[[Dict[str, torch.Tensor]], torch.Tensor],\n    params: Dict[str, torch.Tensor],\n    vec: Dict[str, torch.Tensor],\n    reverse_only: bool = True,\n) -&gt; Dict[str, torch.Tensor]:\n    r\"\"\"\n    Computes the Hessian-vector product (HVP) for a given function at the given\n    parameters, i.e.\n\n    \\[\\nabla_{\\theta} \\nabla_{\\theta} f (\\theta)\\cdot v\\]\n\n    This function can operate in two modes, either reverse-mode autodiff only or both\n    forward- and reverse-mode autodiff.\n\n    Args:\n        func: The scalar-valued function for which the HVP is computed.\n        params: The parameters at which the HVP is computed.\n        vec: The vector with which the Hessian is multiplied.\n        reverse_only: Whether to use only reverse-mode autodiff\n            (True, default) or both forward- and reverse-mode autodiff (False).\n\n    Returns:\n        The HVP of the function at the given parameters with the given vector.\n\n    ??? Example\n\n        ```pycon\n        &gt;&gt;&gt; def f(z): return torch.sum(z**2)\n        &gt;&gt;&gt; u = torch.ones(10, requires_grad=True)\n        &gt;&gt;&gt; v = torch.ones(10)\n        &gt;&gt;&gt; hvp_vec = hvp(f, u, v)\n        &gt;&gt;&gt; assert torch.allclose(hvp_vec, torch.full((10, ), 2.0))\n        ```\n    \"\"\"\n\n    output: Dict[str, torch.Tensor]\n\n    if reverse_only:\n        _, vjp_fn = vjp(grad(func), params)\n        output = vjp_fn(vec)[0]\n    else:\n        output = jvp(grad(func), (params,), (vec,))[1]\n\n    return output\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_batch_hvp_function","title":"create_batch_hvp_function","text":"<pre><code>create_batch_hvp_function(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    reverse_only: bool = True,\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor, Tensor], Tensor]\n</code></pre> <p>Creates a function to compute Hessian-vector product (HVP) for a given model and loss function, where the Hessian information is computed for a provided batch.</p> <p>This function takes a PyTorch model, a loss function, and an optional boolean parameter. It returns a callable that computes the Hessian-vector product for batches of input data and a given vector. The computation can be performed in reverse mode only, based on the <code>reverse_only</code> parameter.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which the Hessian-vector product is to be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>The loss function. It should take two torch.Tensor objects as input and return a torch.Tensor.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>reverse_only</code> <p>If True, the Hessian-vector product is computed in reverse mode only.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor, Tensor], Tensor]</code> <p>A function that takes three <code>torch.Tensor</code> objects - input data (<code>x</code>), target data (<code>y</code>), and a vector (<code>vec</code>), and returns the Hessian-vector product of the loss evaluated on <code>x</code>, <code>y</code> times <code>vec</code>.</p> Example <pre><code># Assume `model` is a PyTorch model and `loss_fn` is a loss function.\nb_hvp_function = batch_hvp(model, loss_fn)\n\n# `x_batch`, `y_batch` are batches of input and target data,\n# and `vec` is a vector.\nhvp_result = b_hvp_function(x_batch, y_batch, vec)\n</code></pre> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_batch_hvp_function(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    reverse_only: bool = True,\n) -&gt; Callable[\n    [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor\n]:\n    r\"\"\"\n    Creates a function to compute Hessian-vector product (HVP) for a given model and\n    loss function, where the Hessian information is computed for a provided batch.\n\n    This function takes a PyTorch model, a loss function,\n    and an optional boolean parameter. It returns a callable\n    that computes the Hessian-vector product for batches of input data\n    and a given vector. The computation can be performed in reverse mode only,\n    based on the `reverse_only` parameter.\n\n    Args:\n        model: The PyTorch model for which the Hessian-vector product is to be computed.\n        loss: The loss function. It should take two\n            torch.Tensor objects as input and return a torch.Tensor.\n        reverse_only (bool, optional): If True, the Hessian-vector product is computed\n            in reverse mode only.\n\n    Returns:\n        A function that takes three `torch.Tensor` objects - input data (`x`),\n            target data (`y`), and a vector (`vec`),\n            and returns the Hessian-vector product of the loss\n            evaluated on `x`, `y` times `vec`.\n\n    ??? Example\n        ```python\n        # Assume `model` is a PyTorch model and `loss_fn` is a loss function.\n        b_hvp_function = batch_hvp(model, loss_fn)\n\n        # `x_batch`, `y_batch` are batches of input and target data,\n        # and `vec` is a vector.\n        hvp_result = b_hvp_function(x_batch, y_batch, vec)\n        ```\n    \"\"\"\n\n    def b_hvp(\n        params: Dict[str, torch.Tensor],\n        x: torch.Tensor,\n        y: torch.Tensor,\n        vec: torch.Tensor,\n    ):\n        return flatten_dimensions(\n            hvp(\n                lambda p: create_batch_loss_function(model, loss)(p, x, y),\n                params,\n                align_structure(params, vec),\n                reverse_only=reverse_only,\n            ).values()\n        )\n\n    return b_hvp\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_empirical_loss_function","title":"create_empirical_loss_function","text":"<pre><code>create_empirical_loss_function(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    data_loader: DataLoader,\n) -&gt; Callable[[Dict[str, Tensor]], Tensor]\n</code></pre> <p>Creates a function to compute the empirical loss of a given model on a given dataset. If we denote the model parameters with \\( \\theta \\), the resulting function approximates:</p> \\[     f(\\theta) = \\frac{1}{N}\\sum_{i=1}^N     \\operatorname{loss}(y_i, \\operatorname{model}(\\theta, x_i)) \\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\), where \\(N\\) is the number of all elements provided by the data_loader.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The model for which the loss should be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>The loss function to be used.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>data_loader</code> <p>The data loader for iterating over the dataset.</p> <p> TYPE: <code>DataLoader</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor]], Tensor]</code> <p>A function that computes the empirical loss of the model on the dataset for given model parameters.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_empirical_loss_function(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    data_loader: DataLoader,\n) -&gt; Callable[[Dict[str, torch.Tensor]], torch.Tensor]:\n    r\"\"\"\n    Creates a function to compute the empirical loss of a given model\n    on a given dataset. If we denote the model parameters with \\( \\theta \\),\n    the resulting function approximates:\n\n    \\[\n        f(\\theta) = \\frac{1}{N}\\sum_{i=1}^N\n        \\operatorname{loss}(y_i, \\operatorname{model}(\\theta, x_i))\n    \\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$\n    with model parameters $\\theta$, where $N$ is the number of all elements provided\n    by the data_loader.\n\n    Args:\n        model: The model for which the loss should be computed.\n        loss: The loss function to be used.\n        data_loader: The data loader for iterating over the dataset.\n\n    Returns:\n        A function that computes the empirical loss of the model on the dataset for\n            given model parameters.\n\n    \"\"\"\n\n    def empirical_loss(params: Dict[str, torch.Tensor]):\n        total_loss = to_model_device(torch.zeros((), requires_grad=True), model)\n        total_samples = to_model_device(torch.zeros(()), model)\n\n        for x, y in iter(data_loader):\n            output = functional_call(\n                model,\n                params,\n                (to_model_device(x, model),),\n            )\n            loss_value = loss(output, to_model_device(y, model))\n            total_loss = total_loss + loss_value * x.size(0)\n            total_samples += x.size(0)\n\n        return total_loss / total_samples\n\n    return empirical_loss\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_batch_loss_function","title":"create_batch_loss_function","text":"<pre><code>create_batch_loss_function(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor]\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]\n</code></pre> <p>Creates a function to compute the loss of a given model on a given batch of data, i.e. the function</p> \\[f(\\theta, x, y) = \\frac{1}{N} \\sum_{i=1}^N     \\operatorname{loss}(\\operatorname{model}(\\theta, x_i), y_i)\\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\), where \\(N\\) is the number of elements in the batch. Args:     model: The model for which the loss should be computed.     loss: The loss function to be used, which should be able to handle         a batch dimension</p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]</code> <p>A function that computes the loss of the model on a batch for given model parameters. The model parameter input to the function must take the form of a dict conform to model.named_parameters(), i.e. the keys must be a subset of the parameters and the corresponding tensor shapes must align. For the data input, the first dimension has to be the batch dimension.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_batch_loss_function(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n) -&gt; Callable[[Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], torch.Tensor]:\n    r\"\"\"\n    Creates a function to compute the loss of a given model on a given batch of data,\n    i.e. the function\n\n    \\[f(\\theta, x, y) = \\frac{1}{N} \\sum_{i=1}^N\n        \\operatorname{loss}(\\operatorname{model}(\\theta, x_i), y_i)\\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$\n    with model parameters $\\theta$, where $N$ is the number of elements in the batch.\n    Args:\n        model: The model for which the loss should be computed.\n        loss: The loss function to be used, which should be able to handle\n            a batch dimension\n\n    Returns:\n        A function that computes the loss of the model on a batch for given\n            model parameters. The model parameter input to the function must take\n            the form of a dict conform to model.named_parameters(), i.e. the keys\n            must be a subset of the parameters and the corresponding tensor shapes\n            must align. For the data input, the first dimension has to be the batch\n            dimension.\n    \"\"\"\n\n    def batch_loss(params: Dict[str, torch.Tensor], x: torch.Tensor, y: torch.Tensor):\n        outputs = functional_call(model, params, (to_model_device(x, model),))\n        return loss(outputs, y)\n\n    return batch_loss\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_hvp_function","title":"create_hvp_function","text":"<pre><code>create_hvp_function(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    data_loader: DataLoader,\n    precompute_grad: bool = True,\n    use_average: bool = True,\n    reverse_only: bool = True,\n    track_gradients: bool = False,\n) -&gt; Callable[[Tensor], Tensor]\n</code></pre> <p>Returns a function that calculates the approximate Hessian-vector product for a given vector. If you want to compute the exact hessian, i.e., pulling all data into memory and compute a full gradient computation, use the function hvp.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch module representing the model whose loss function's Hessian is to be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>data_loader</code> <p>A DataLoader instance that provides batches of data for calculating the Hessian-vector product. Each batch from the DataLoader is assumed to return a tuple where the first element is the model's input and the second element is the target output.</p> <p> TYPE: <code>DataLoader</code> </p> <code>precompute_grad</code> <p>If <code>True</code>, the full data gradient is precomputed and kept in memory, which can speed up the hessian vector product computation. Set this to <code>False</code>, if you can't afford to keep the full computation graph in memory.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>use_average</code> <p>If <code>True</code>, the returned function uses batch-wise computation via a batch loss function and averages the results. If <code>False</code>, the function uses backpropagation on the full empirical loss function, which is more accurate than averaging the batch hessians, but probably has a way higher memory usage.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>reverse_only</code> <p>Whether to use only reverse-mode autodiff or both forward- and reverse-mode autodiff. Ignored if <code>precompute_grad</code> is <code>True</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>track_gradients</code> <p>Whether to track gradients for the resulting tensor of the Hessian-vector products.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Callable[[Tensor], Tensor]</code> <p>A function that takes a single argument, a vector, and returns the</p> <code>Callable[[Tensor], Tensor]</code> <p>product of the Hessian of the <code>loss</code> function with respect to the</p> <code>Callable[[Tensor], Tensor]</code> <p><code>model</code>'s parameters and the input vector.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_hvp_function(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    data_loader: DataLoader,\n    precompute_grad: bool = True,\n    use_average: bool = True,\n    reverse_only: bool = True,\n    track_gradients: bool = False,\n) -&gt; Callable[[torch.Tensor], torch.Tensor]:\n    \"\"\"\n    Returns a function that calculates the approximate Hessian-vector product\n    for a given vector. If you want to compute the exact hessian,\n    i.e., pulling all data into memory and compute a full gradient computation, use\n    the function [hvp][pydvl.influence.torch.functional.hvp].\n\n    Args:\n        model: A PyTorch module representing the model whose loss function's\n            Hessian is to be computed.\n        loss: A callable that takes the model's output and target as input and\n            returns the scalar loss.\n        data_loader: A DataLoader instance that provides batches of data for\n            calculating the Hessian-vector product. Each batch from the\n            DataLoader is assumed to return a tuple where the first element is\n            the model's input and the second element is the target output.\n        precompute_grad: If `True`, the full data gradient is precomputed and\n            kept in memory, which can speed up the hessian vector product\n            computation. Set this to `False`, if you can't afford to keep the\n            full computation graph in memory.\n        use_average: If `True`, the returned function uses batch-wise\n            computation via\n            [a batch loss function][pydvl.influence.torch.functional.create_batch_loss_function]\n            and averages the results.\n            If `False`, the function uses backpropagation on the full\n            [empirical loss function]\n            [pydvl.influence.torch.functional.create_empirical_loss_function],\n            which is more accurate than averaging the batch hessians, but\n            probably has a way higher memory usage.\n        reverse_only: Whether to use only reverse-mode autodiff or\n            both forward- and reverse-mode autodiff. Ignored if\n            `precompute_grad` is `True`.\n        track_gradients: Whether to track gradients for the resulting tensor of\n            the Hessian-vector products.\n\n    Returns:\n        A function that takes a single argument, a vector, and returns the\n        product of the Hessian of the `loss` function with respect to the\n        `model`'s parameters and the input vector.\n    \"\"\"\n\n    if precompute_grad:\n        model_params = {k: p for k, p in model.named_parameters() if p.requires_grad}\n\n        if use_average:\n            model_dtype = next(p.dtype for p in model.parameters() if p.requires_grad)\n            total_grad_xy = torch.empty(0, dtype=model_dtype)\n            total_points = 0\n            grad_func = torch.func.grad(create_batch_loss_function(model, loss))\n            for x, y in iter(data_loader):\n                grad_xy = grad_func(\n                    model_params, to_model_device(x, model), to_model_device(y, model)\n                )\n                grad_xy = flatten_dimensions(grad_xy.values())\n                if total_grad_xy.nelement() == 0:\n                    total_grad_xy = torch.zeros_like(grad_xy)\n                total_grad_xy += grad_xy * len(x)\n                total_points += len(x)\n            total_grad_xy /= total_points\n        else:\n            total_grad_xy = torch.func.grad(\n                create_empirical_loss_function(model, loss, data_loader)\n            )(model_params)\n            total_grad_xy = flatten_dimensions(total_grad_xy.values())\n\n        def precomputed_grads_hvp_function(\n            precomputed_grads: torch.Tensor, vec: torch.Tensor\n        ) -&gt; torch.Tensor:\n            vec = to_model_device(vec, model)\n            if vec.ndim == 1:\n                vec = vec.unsqueeze(0)\n\n            z = (precomputed_grads * torch.autograd.Variable(vec)).sum(dim=1)\n\n            mvp = []\n            for i in range(len(z)):\n                mvp.append(\n                    flatten_dimensions(\n                        torch.autograd.grad(\n                            z[i], list(model_params.values()), retain_graph=True\n                        )\n                    )\n                )\n            result = torch.stack([arr.contiguous().view(-1) for arr in mvp])\n\n            if not track_gradients:\n                result = result.detach()\n\n            return result\n\n        return partial(precomputed_grads_hvp_function, total_grad_xy)\n\n    def hvp_function(vec: torch.Tensor) -&gt; torch.Tensor:\n        params = {\n            k: p if track_gradients else p.detach()\n            for k, p in model.named_parameters()\n            if p.requires_grad\n        }\n        v = align_structure(params, vec)\n        empirical_loss = create_empirical_loss_function(model, loss, data_loader)\n        return flatten_dimensions(\n            hvp(empirical_loss, params, v, reverse_only=reverse_only).values()\n        )\n\n    def avg_hvp_function(vec: torch.Tensor) -&gt; torch.Tensor:\n        n_batches = len(data_loader)\n        avg_hessian = to_model_device(torch.zeros_like(vec), model)\n        b_hvp = create_batch_hvp_function(model, loss, reverse_only)\n        params = {\n            k: p if track_gradients else p.detach()\n            for k, p in model.named_parameters()\n            if p.requires_grad\n        }\n        for t_x, t_y in iter(data_loader):\n            t_x, t_y = to_model_device(t_x, model), to_model_device(t_y, model)\n            avg_hessian += b_hvp(params, t_x, t_y, to_model_device(vec, model))\n\n        return avg_hessian / float(n_batches)\n\n    return avg_hvp_function if use_average else hvp_function\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.hessian","title":"hessian","text":"<pre><code>hessian(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    data_loader: DataLoader,\n    use_hessian_avg: bool = True,\n    track_gradients: bool = False,\n) -&gt; Tensor\n</code></pre> <p>Computes the Hessian matrix for a given model and loss function.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which the Hessian is computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>data_loader</code> <p>DataLoader providing batches of input data and corresponding ground truths.</p> <p> TYPE: <code>DataLoader</code> </p> <code>use_hessian_avg</code> <p>Flag to indicate whether the average Hessian across mini-batches should be computed. If False, the empirical loss across the entire dataset is used.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>track_gradients</code> <p>Whether to track gradients for the resulting tensor of the hessian vector products.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A tensor representing the Hessian matrix. The shape of the tensor will be (n_parameters, n_parameters), where n_parameters is the number of trainable parameters in the model.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def hessian(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    data_loader: DataLoader,\n    use_hessian_avg: bool = True,\n    track_gradients: bool = False,\n) -&gt; torch.Tensor:\n    \"\"\"\n    Computes the Hessian matrix for a given model and loss function.\n\n    Args:\n        model: The PyTorch model for which the Hessian is computed.\n        loss: A callable that computes the loss.\n        data_loader: DataLoader providing batches of input data and corresponding\n            ground truths.\n        use_hessian_avg: Flag to indicate whether the average Hessian across\n            mini-batches should be computed.\n            If False, the empirical loss across the entire dataset is used.\n        track_gradients: Whether to track gradients for the resulting tensor of\n            the hessian vector products.\n\n    Returns:\n        A tensor representing the Hessian matrix. The shape of the tensor will be\n            (n_parameters, n_parameters), where n_parameters is the number of trainable\n            parameters in the model.\n    \"\"\"\n\n    params = {\n        k: p if track_gradients else p.detach()\n        for k, p in model.named_parameters()\n        if p.requires_grad\n    }\n    n_parameters = sum([p.numel() for p in params.values()])\n    model_dtype = next((p.dtype for p in params.values()))\n\n    flat_params = flatten_dimensions(params.values())\n\n    if use_hessian_avg:\n        n_samples = 0\n        hessian_mat = to_model_device(\n            torch.zeros((n_parameters, n_parameters), dtype=model_dtype), model\n        )\n        blf = create_batch_loss_function(model, loss)\n\n        def flat_input_batch_loss_function(\n            p: torch.Tensor, t_x: torch.Tensor, t_y: torch.Tensor\n        ):\n            return blf(align_with_model(p, model), t_x, t_y)\n\n        for x, y in iter(data_loader):\n            n_samples += x.shape[0]\n            hessian_mat += x.shape[0] * torch.func.hessian(\n                flat_input_batch_loss_function\n            )(flat_params, to_model_device(x, model), to_model_device(y, model))\n\n        hessian_mat /= n_samples\n    else:\n\n        def flat_input_empirical_loss(p: torch.Tensor):\n            return create_empirical_loss_function(model, loss, data_loader)(\n                align_with_model(p, model)\n            )\n\n        hessian_mat = torch.func.jacrev(torch.func.jacrev(flat_input_empirical_loss))(\n            flat_params\n        )\n\n    return hessian_mat\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_per_sample_loss_function","title":"create_per_sample_loss_function","text":"<pre><code>create_per_sample_loss_function(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor]\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]\n</code></pre> <p>Generates a function to compute per-sample losses using PyTorch's vmap, i.e. the vector-valued function</p> \\[ f(\\theta, x, y)  = (\\operatorname{loss}(\\operatorname{model}(\\theta, x_1), y_1),     \\dots,     \\operatorname{loss}(\\operatorname{model}(\\theta, x_N), y_N)), \\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\), where \\(N\\) is the number of elements in the batch.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which per-sample losses will be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]</code> <p>A callable that computes the loss for each sample in the batch, given a dictionary of model inputs, the model's predictions, and the true values. The callable will return a tensor where each entry corresponds to the loss of the corresponding sample.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_per_sample_loss_function(\n    model: torch.nn.Module, loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor]\n) -&gt; Callable[[Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], torch.Tensor]:\n    r\"\"\"\n    Generates a function to compute per-sample losses using PyTorch's vmap,\n    i.e. the vector-valued function\n\n    \\[ f(\\theta, x, y)  = (\\operatorname{loss}(\\operatorname{model}(\\theta, x_1), y_1),\n        \\dots,\n        \\operatorname{loss}(\\operatorname{model}(\\theta, x_N), y_N)), \\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$ with\n    model parameters $\\theta$, where $N$ is the number of elements in the batch.\n\n    Args:\n        model: The PyTorch model for which per-sample losses will be computed.\n        loss: A callable that computes the loss.\n\n    Returns:\n        A callable that computes the loss for each sample in the batch,\n            given a dictionary of model inputs, the model's predictions,\n            and the true values. The callable will return a tensor where\n            each entry corresponds to the loss of the corresponding sample.\n    \"\"\"\n\n    def compute_loss(\n        params: Dict[str, torch.Tensor], x: torch.Tensor, y: torch.Tensor\n    ) -&gt; torch.Tensor:\n        outputs = functional_call(\n            model, params, (to_model_device(x.unsqueeze(0), model),)\n        )\n        return loss(outputs, y.unsqueeze(0))\n\n    vmap_loss: Callable[\n        [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], torch.Tensor\n    ] = torch.vmap(compute_loss, in_dims=(None, 0, 0))\n    return vmap_loss\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_per_sample_gradient_function","title":"create_per_sample_gradient_function","text":"<pre><code>create_per_sample_gradient_function(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor]\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor], Dict[str, Tensor]]\n</code></pre> <p>Generates a function to computes the per-sample gradient of the loss with respect to the model's parameters, i.e. the tensor-valued function</p> \\[ f(\\theta, x, y) = (\\nabla_{\\theta}\\operatorname{loss}     (\\operatorname{model}(\\theta, x_1), y_1), \\dots,     \\nabla_{\\theta}\\operatorname{loss}(\\operatorname{model}(\\theta, x_N), y_N) \\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\), where \\(N\\) is the number of elements in the batch.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which per-sample gradients will be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor], Dict[str, Tensor]]</code> <p>A callable that takes a dictionary of model parameters, the model's input, and the labels. It returns a dictionary with the same keys as the model's named parameters. Each entry in the returned dictionary corresponds to the gradient of the corresponding model parameter for each sample in the batch.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_per_sample_gradient_function(\n    model: torch.nn.Module, loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor]\n) -&gt; Callable[\n    [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], Dict[str, torch.Tensor]\n]:\n    r\"\"\"\n    Generates a function to computes the per-sample gradient of the loss with respect to\n    the model's parameters, i.e. the tensor-valued function\n\n    \\[ f(\\theta, x, y) = (\\nabla_{\\theta}\\operatorname{loss}\n        (\\operatorname{model}(\\theta, x_1), y_1), \\dots,\n        \\nabla_{\\theta}\\operatorname{loss}(\\operatorname{model}(\\theta, x_N), y_N) \\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$ with\n    model parameters $\\theta$, where $N$ is the number of elements in the batch.\n\n    Args:\n        model: The PyTorch model for which per-sample gradients will be computed.\n        loss: A callable that computes the loss.\n\n    Returns:\n        A callable that takes a dictionary of model parameters, the model's input,\n            and the labels. It returns a dictionary with the same keys as the model's\n            named parameters. Each entry in the returned dictionary corresponds to\n            the gradient of the corresponding model parameter for each sample\n            in the batch.\n\n    \"\"\"\n\n    per_sample_grad: Callable[\n        [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], Dict[str, torch.Tensor]\n    ] = torch.func.jacrev(create_per_sample_loss_function(model, loss))\n    return per_sample_grad\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_matrix_jacobian_product_function","title":"create_matrix_jacobian_product_function","text":"<pre><code>create_matrix_jacobian_product_function(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor], g: Tensor\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]\n</code></pre> <p>Generates a function to computes the matrix-Jacobian product (MJP) of the per-sample loss with respect to the model's parameters, i.e. the function</p> \\[ f(\\theta, x, y) = g \\, @ \\, (\\nabla_{\\theta}\\operatorname{loss}     (\\operatorname{model}(\\theta, x_i), y_i))_i^T \\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\).</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which the MJP will be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>g</code> <p>Matrix for which the product with the Jacobian will be computed. The shape of this matrix should be consistent with the shape of the jacobian.</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]</code> <p>A callable that takes a dictionary of model inputs, the model's input, and the labels. The callable returns the matrix-Jacobian product of the per-sample loss with respect to the model's parameters for the given matrix <code>g</code>.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_matrix_jacobian_product_function(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    g: torch.Tensor,\n) -&gt; Callable[[Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], torch.Tensor]:\n    r\"\"\"\n    Generates a function to computes the matrix-Jacobian product (MJP) of the\n    per-sample loss with respect to the model's parameters, i.e. the function\n\n    \\[ f(\\theta, x, y) = g \\, @ \\, (\\nabla_{\\theta}\\operatorname{loss}\n        (\\operatorname{model}(\\theta, x_i), y_i))_i^T \\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$ with\n    model parameters $\\theta$.\n\n    Args:\n        model: The PyTorch model for which the MJP will be computed.\n        loss: A callable that computes the loss.\n        g: Matrix for which the product with the Jacobian will be computed.\n            The shape of this matrix should be consistent with the shape of\n            the jacobian.\n\n    Returns:\n        A callable that takes a dictionary of model inputs, the model's input,\n            and the labels. The callable returns the matrix-Jacobian product of the\n            per-sample loss with respect to the model's parameters for the given\n            matrix `g`.\n\n    \"\"\"\n\n    def single_jvp(\n        params: Dict[str, torch.Tensor],\n        x: torch.Tensor,\n        y: torch.Tensor,\n        _g: torch.Tensor,\n    ):\n        return torch.func.jvp(\n            lambda p: create_per_sample_loss_function(model, loss)(p, x, y),\n            (params,),\n            (align_with_model(_g, model),),\n        )[1]\n\n    def full_jvp(params: Dict[str, torch.Tensor], x: torch.Tensor, y: torch.Tensor):\n        return torch.func.vmap(single_jvp, in_dims=(None, None, None, 0))(\n            params, x, y, g\n        )\n\n    return full_jvp\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_per_sample_mixed_derivative_function","title":"create_per_sample_mixed_derivative_function","text":"<pre><code>create_per_sample_mixed_derivative_function(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor]\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor], Dict[str, Tensor]]\n</code></pre> <p>Generates a function to computes the mixed derivatives, of the per-sample loss with respect to the model parameters and the input, i.e. the function</p> \\[ f(\\theta, x, y) = \\nabla_{\\theta}\\nabla_{x}\\operatorname{loss}     (\\operatorname{model}(\\theta, x), y) \\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\).</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which the mixed derivatives are computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor], Dict[str, Tensor]]</code> <p>A callable that takes a dictionary of model inputs, the model's input, and the labels. The callable returns the mixed derivatives of the per-sample loss with respect to the model's parameters and input.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_per_sample_mixed_derivative_function(\n    model: torch.nn.Module, loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor]\n) -&gt; Callable[\n    [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], Dict[str, torch.Tensor]\n]:\n    r\"\"\"\n    Generates a function to computes the mixed derivatives, of the per-sample loss with\n    respect to the model parameters and the input, i.e. the function\n\n    \\[ f(\\theta, x, y) = \\nabla_{\\theta}\\nabla_{x}\\operatorname{loss}\n        (\\operatorname{model}(\\theta, x), y) \\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$ with\n    model parameters $\\theta$.\n\n    Args:\n        model: The PyTorch model for which the mixed derivatives are computed.\n        loss: A callable that computes the loss.\n\n    Returns:\n        A callable that takes a dictionary of model inputs, the model's input,\n            and the labels. The callable returns the mixed derivatives of the\n            per-sample loss with respect to the model's parameters and input.\n\n    \"\"\"\n\n    def compute_loss(params: Dict[str, torch.Tensor], x: torch.Tensor, y: torch.Tensor):\n        outputs = functional_call(\n            model, params, (to_model_device(x.unsqueeze(0), model),)\n        )\n        return loss(outputs, y.unsqueeze(0))\n\n    per_samp_mix_derivative: Callable[\n        [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], Dict[str, torch.Tensor]\n    ] = torch.vmap(\n        torch.func.jacrev(torch.func.grad(compute_loss, argnums=1)),\n        in_dims=(None, 0, 0),\n    )\n    return per_samp_mix_derivative\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.lanzcos_low_rank_hessian_approx","title":"lanzcos_low_rank_hessian_approx","text":"<pre><code>lanzcos_low_rank_hessian_approx(\n    hessian_vp: Callable[[Tensor], Tensor],\n    matrix_shape: Tuple[int, int],\n    hessian_perturbation: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-06,\n    max_iter: Optional[int] = None,\n    device: Optional[device] = None,\n    eigen_computation_on_gpu: bool = False,\n    torch_dtype: Optional[dtype] = None,\n) -&gt; LowRankProductRepresentation\n</code></pre> <p>Calculates a low-rank approximation of the Hessian matrix of a scalar-valued function using the implicitly restarted Lanczos algorithm, i.e.:</p> \\[ H_{\\text{approx}} = V D V^T\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.</p> PARAMETER  DESCRIPTION <code>hessian_vp</code> <p>A function that takes a vector and returns the product of the Hessian of the loss function.</p> <p> TYPE: <code>Callable[[Tensor], Tensor]</code> </p> <code>matrix_shape</code> <p>The shape of the matrix, represented by the hessian vector product.</p> <p> TYPE: <code>Tuple[int, int]</code> </p> <code>hessian_perturbation</code> <p>Regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. If not provided, it defaults to \\( \\min(\\text{model.n_parameters},     \\max(2 \\times \\text{rank_estimate} + 1, 20)) \\).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm, which stops when the difference in the approximated eigenvalue is less than <code>tol</code>. Defaults to 1e-6.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. If not provided, it defaults to \\( 10 \\cdot \\text{model.n_parameters}\\).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>device</code> <p>The device to use for executing the hessian vector product.</p> <p> TYPE: <code>Optional[device]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the provided device via cupy implementation. Ensure that either your model is small enough, or you use a small rank_estimate to fit your device's memory. If False, the eigen pair approximation is executed on the CPU with scipy's wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>torch_dtype</code> <p>If not provided, the current torch default dtype is used for conversion to torch.</p> <p> TYPE: <code>Optional[dtype]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>LowRankProductRepresentation instance that contains the top (up until rank_estimate) eigenvalues and corresponding eigenvectors of the Hessian.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def lanzcos_low_rank_hessian_approx(\n    hessian_vp: Callable[[torch.Tensor], torch.Tensor],\n    matrix_shape: Tuple[int, int],\n    hessian_perturbation: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-6,\n    max_iter: Optional[int] = None,\n    device: Optional[torch.device] = None,\n    eigen_computation_on_gpu: bool = False,\n    torch_dtype: Optional[torch.dtype] = None,\n) -&gt; LowRankProductRepresentation:\n    r\"\"\"\n    Calculates a low-rank approximation of the Hessian matrix of a scalar-valued\n    function using the implicitly restarted Lanczos algorithm, i.e.:\n\n    \\[ H_{\\text{approx}} = V D V^T\\]\n\n    where \\(D\\) is a diagonal matrix with the top (in absolute value) `rank_estimate`\n    eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.\n\n    Args:\n        hessian_vp: A function that takes a vector and returns the product of\n            the Hessian of the loss function.\n        matrix_shape: The shape of the matrix, represented by the hessian vector\n            product.\n        hessian_perturbation: Regularization parameter added to the\n            Hessian-vector product for numerical stability.\n        rank_estimate: The number of eigenvalues and corresponding eigenvectors\n            to compute. Represents the desired rank of the Hessian approximation.\n        krylov_dimension: The number of Krylov vectors to use for the Lanczos\n            method. If not provided, it defaults to\n            \\( \\min(\\text{model.n_parameters},\n                \\max(2 \\times \\text{rank_estimate} + 1, 20)) \\).\n        tol: The stopping criteria for the Lanczos algorithm, which stops when\n            the difference in the approximated eigenvalue is less than `tol`.\n            Defaults to 1e-6.\n        max_iter: The maximum number of iterations for the Lanczos method. If\n            not provided, it defaults to \\( 10 \\cdot \\text{model.n_parameters}\\).\n        device: The device to use for executing the hessian vector product.\n        eigen_computation_on_gpu: If True, tries to execute the eigen pair\n            approximation on the provided device via [cupy](https://cupy.dev/)\n            implementation. Ensure that either your model is small enough, or you\n            use a small rank_estimate to fit your device's memory. If False, the\n            eigen pair approximation is executed on the CPU with scipy's wrapper to\n            ARPACK.\n        torch_dtype: If not provided, the current torch default dtype is used for\n            conversion to torch.\n\n    Returns:\n        [LowRankProductRepresentation]\n            [pydvl.influence.torch.functional.LowRankProductRepresentation]\n            instance that contains the top (up until rank_estimate) eigenvalues\n            and corresponding eigenvectors of the Hessian.\n    \"\"\"\n\n    torch_dtype = torch.get_default_dtype() if torch_dtype is None else torch_dtype\n\n    if eigen_computation_on_gpu:\n        try:\n            import cupy as cp\n            from cupyx.scipy.sparse.linalg import LinearOperator, eigsh\n            from torch.utils.dlpack import from_dlpack, to_dlpack\n        except ImportError as e:\n            raise ImportError(\n                f\"Try to install missing dependencies or set eigen_computation_on_gpu \"\n                f\"to False: {e}\"\n            )\n\n        if device is None:\n            raise ValueError(\n                \"Without setting an explicit device, cupy is not supported\"\n            )\n\n        def to_torch_conversion_function(x: cp.NDArray) -&gt; torch.Tensor:\n            return from_dlpack(x.toDlpack()).to(torch_dtype)\n\n        def mv(x):\n            x = to_torch_conversion_function(x)\n            y = hessian_vp(x) + hessian_perturbation * x\n            return cp.from_dlpack(to_dlpack(y))\n\n    else:\n        from scipy.sparse.linalg import LinearOperator, eigsh\n\n        def mv(x):\n            x_torch = torch.as_tensor(x, device=device, dtype=torch_dtype)\n            y = (\n                (hessian_vp(x_torch) + hessian_perturbation * x_torch)\n                .detach()\n                .cpu()\n                .numpy()\n            )\n            return y\n\n        to_torch_conversion_function = partial(torch.as_tensor, dtype=torch_dtype)\n\n    try:\n        eigen_vals, eigen_vecs = eigsh(\n            LinearOperator(matrix_shape, matvec=mv),\n            k=rank_estimate,\n            maxiter=max_iter,\n            tol=tol,\n            ncv=krylov_dimension,\n            return_eigenvectors=True,\n        )\n\n    except ArpackNoConvergence as e:\n        logger.warning(\n            f\"ARPACK did not converge for parameters {max_iter=}, {tol=}, \"\n            f\"{krylov_dimension=}, {rank_estimate=}. \\n \"\n            f\"Returning the best approximation found so far. \"\n            f\"Use those with care or modify parameters.\\n Original error: {e}\"\n        )\n\n        eigen_vals, eigen_vecs = e.eigenvalues, e.eigenvectors\n\n    eigen_vals = to_torch_conversion_function(eigen_vals)\n    eigen_vecs = to_torch_conversion_function(eigen_vecs)\n\n    return LowRankProductRepresentation(eigen_vals, eigen_vecs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.model_hessian_low_rank","title":"model_hessian_low_rank","text":"<pre><code>model_hessian_low_rank(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    training_data: DataLoader,\n    hessian_perturbation: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-06,\n    max_iter: Optional[int] = None,\n    eigen_computation_on_gpu: bool = False,\n    precompute_grad: bool = False,\n) -&gt; LowRankProductRepresentation\n</code></pre> <p>Calculates a low-rank approximation of the Hessian matrix of the model's loss function using the implicitly restarted Lanczos algorithm, i.e.</p> \\[ H_{\\text{approx}} = V D V^T\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model instance. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> </p> <code>training_data</code> <p>A DataLoader instance that provides the model's training data. Used in calculating the Hessian-vector products.</p> <p> TYPE: <code>DataLoader</code> </p> <code>hessian_perturbation</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. If not provided, it defaults to min(model.n_parameters,     max(2*rank_estimate + 1, 20)).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm, which stops when the difference in the approximated eigenvalue is less than <code>tol</code>. Defaults to 1e-6.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. If not provided, it defaults to 10*model.n_parameters.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the provided device via cupy implementation. Make sure, that either your model is small enough or you use a small rank_estimate to fit your device's memory. If False, the eigen pair approximation is executed on the CPU by scipy wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>precompute_grad</code> <p>If True, the full data gradient is precomputed and kept in memory, which can speed up the hessian vector product computation. Set this to False, if you can't afford to keep the full computation graph in memory.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>LowRankProductRepresentation instance that contains the top (up until rank_estimate) eigenvalues and corresponding eigenvectors of the Hessian.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def model_hessian_low_rank(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    training_data: DataLoader,\n    hessian_perturbation: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-6,\n    max_iter: Optional[int] = None,\n    eigen_computation_on_gpu: bool = False,\n    precompute_grad: bool = False,\n) -&gt; LowRankProductRepresentation:\n    r\"\"\"\n    Calculates a low-rank approximation of the Hessian matrix of the model's\n    loss function using the implicitly restarted Lanczos algorithm, i.e.\n\n    \\[ H_{\\text{approx}} = V D V^T\\]\n\n    where \\(D\\) is a diagonal matrix with the top (in absolute value) `rank_estimate`\n    eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.\n\n\n    Args:\n        model: A PyTorch model instance. The Hessian will be calculated with respect to\n            this model's parameters.\n        loss : A callable that computes the loss.\n        training_data: A DataLoader instance that provides the model's training data.\n            Used in calculating the Hessian-vector products.\n        hessian_perturbation: Optional regularization parameter added to the\n            Hessian-vector product for numerical stability.\n        rank_estimate: The number of eigenvalues and corresponding eigenvectors to\n            compute. Represents the desired rank of the Hessian approximation.\n        krylov_dimension: The number of Krylov vectors to use for the Lanczos method.\n            If not provided, it defaults to min(model.n_parameters,\n                max(2*rank_estimate + 1, 20)).\n        tol: The stopping criteria for the Lanczos algorithm,\n            which stops when the difference in the approximated eigenvalue is less than\n            `tol`. Defaults to 1e-6.\n        max_iter: The maximum number of iterations for the Lanczos method.\n            If not provided, it defaults to 10*model.n_parameters.\n        eigen_computation_on_gpu: If True, tries to execute the eigen pair approximation\n            on the provided device via cupy implementation.\n            Make sure, that either your model is small enough or you use a\n            small rank_estimate to fit your device's memory.\n            If False, the eigen pair approximation is executed on the CPU by\n            scipy wrapper to ARPACK.\n        precompute_grad: If True, the full data gradient is precomputed and kept\n            in memory, which can speed up the hessian vector product computation.\n            Set this to False, if you can't afford to keep the full computation graph\n            in memory.\n\n    Returns:\n        [LowRankProductRepresentation]\n            [pydvl.influence.torch.functional.LowRankProductRepresentation]\n            instance that contains the top (up until rank_estimate) eigenvalues\n            and corresponding eigenvectors of the Hessian.\n    \"\"\"\n    raw_hvp = create_hvp_function(\n        model, loss, training_data, use_average=True, precompute_grad=precompute_grad\n    )\n    n_params = sum([p.numel() for p in model.parameters() if p.requires_grad])\n    device = next(model.parameters()).device\n    return lanzcos_low_rank_hessian_approx(\n        hessian_vp=raw_hvp,\n        matrix_shape=(n_params, n_params),\n        hessian_perturbation=hessian_perturbation,\n        rank_estimate=rank_estimate,\n        krylov_dimension=krylov_dimension,\n        tol=tol,\n        max_iter=max_iter,\n        device=device,\n        eigen_computation_on_gpu=eigen_computation_on_gpu,\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.randomized_nystroem_approximation","title":"randomized_nystroem_approximation","text":"<pre><code>randomized_nystroem_approximation(\n    mat_mat_prod: Union[Tensor, Callable[[Tensor], Tensor]],\n    input_dim: int,\n    rank: int,\n    input_type: dtype,\n    shift_func: Optional[Callable[[Tensor], Tensor]] = None,\n    mat_vec_device: device = torch.device(\"cpu\"),\n) -&gt; LowRankProductRepresentation\n</code></pre> <p>Given a matrix vector product function (representing a symmetric positive definite matrix \\(A\\) ), computes a random Nystr\u00f6m low rank approximation of \\(A\\) in factored form, i.e.</p> \\[ A_{\\text{nys}} = (A \\Omega)(\\Omega^T A \\Omega)^{\\dagger}(A \\Omega)^T = U \\Sigma U^T \\] <p>where \\(\\Omega\\) is a standard normal random matrix.</p> PARAMETER  DESCRIPTION <code>mat_mat_prod</code> <p>A callable representing the matrix vector product</p> <p> TYPE: <code>Union[Tensor, Callable[[Tensor], Tensor]]</code> </p> <code>input_dim</code> <p>dimension of the input for the matrix vector product</p> <p> TYPE: <code>int</code> </p> <code>input_type</code> <p>data_type of inputs</p> <p> TYPE: <code>dtype</code> </p> <code>rank</code> <p>rank of the approximation</p> <p> TYPE: <code>int</code> </p> <code>shift_func</code> <p>optional function for computing the stabilizing shift in the construction of the randomized nystroem approximation, defaults to</p> \\[ \\sqrt{\\operatorname{\\text{input_dim}}} \\cdot     \\varepsilon(\\operatorname{\\text{input_type}}) \\cdot \\|A\\Omega\\|_2,\\] <p>where \\(\\varepsilon(\\operatorname{\\text{input_type}})\\) is the value of the machine precision corresponding to the data type.</p> <p> TYPE: <code>Optional[Callable[[Tensor], Tensor]]</code> DEFAULT: <code>None</code> </p> <code>mat_vec_device</code> <p>device where the matrix vector product has to be executed</p> <p> TYPE: <code>device</code> DEFAULT: <code>device('cpu')</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>object containing, \\(U\\) and \\(\\Sigma\\)</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def randomized_nystroem_approximation(\n    mat_mat_prod: Union[torch.Tensor, Callable[[torch.Tensor], torch.Tensor]],\n    input_dim: int,\n    rank: int,\n    input_type: torch.dtype,\n    shift_func: Optional[Callable[[torch.Tensor], torch.Tensor]] = None,\n    mat_vec_device: torch.device = torch.device(\"cpu\"),\n) -&gt; LowRankProductRepresentation:\n    r\"\"\"\n    Given a matrix vector product function (representing a symmetric positive definite\n    matrix $A$ ), computes a random Nystr\u00f6m low rank approximation of\n    $A$ in factored form, i.e.\n\n    $$ A_{\\text{nys}} = (A \\Omega)(\\Omega^T A \\Omega)^{\\dagger}(A \\Omega)^T\n    = U \\Sigma U^T $$\n\n    where $\\Omega$ is a standard normal random matrix.\n\n    Args:\n        mat_mat_prod: A callable representing the matrix vector product\n        input_dim: dimension of the input for the matrix vector product\n        input_type: data_type of inputs\n        rank: rank of the approximation\n        shift_func: optional function for computing the stabilizing shift in the\n            construction of the randomized nystroem approximation, defaults to\n\n            $$ \\sqrt{\\operatorname{\\text{input_dim}}} \\cdot\n                \\varepsilon(\\operatorname{\\text{input_type}}) \\cdot \\|A\\Omega\\|_2,$$\n\n            where $\\varepsilon(\\operatorname{\\text{input_type}})$ is the value of the\n            machine precision corresponding to the data type.\n        mat_vec_device: device where the matrix vector product has to be executed\n\n    Returns:\n        object containing, $U$ and $\\Sigma$\n    \"\"\"\n\n    if shift_func is None:\n\n        def shift_func(x: torch.Tensor):\n            return (\n                torch.sqrt(torch.as_tensor(input_dim))\n                * torch.finfo(x.dtype).eps\n                * torch.linalg.norm(x)\n            )\n\n    _mat_mat_prod: Callable[[torch.Tensor], torch.Tensor]\n\n    if isinstance(mat_mat_prod, torch.Tensor):\n\n        def _mat_mat_prod(x: torch.Tensor):\n            return mat_mat_prod @ x\n\n    else:\n        _mat_mat_prod = mat_mat_prod\n\n    random_sample_matrix = torch.randn(\n        input_dim, rank, device=mat_vec_device, dtype=input_type\n    )\n    random_sample_matrix, _ = torch.linalg.qr(random_sample_matrix)\n\n    sketch_mat = _mat_mat_prod(random_sample_matrix)\n\n    shift = shift_func(sketch_mat)\n    sketch_mat += shift * random_sample_matrix\n    cholesky_mat = torch.matmul(random_sample_matrix.t(), sketch_mat)\n    try:\n        triangular_mat = torch.linalg.cholesky(cholesky_mat)\n    except _LinAlgError as e:\n        logger.warning(\n            f\"Encountered error in cholesky decomposition: {e}.\\n \"\n            f\"Increasing shift by smallest eigenvalue and re-compute\"\n        )\n        eigen_vals, eigen_vectors = torch.linalg.eigh(cholesky_mat)\n        shift += torch.abs(torch.min(eigen_vals))\n        eigen_vals += shift\n        triangular_mat = torch.linalg.cholesky(\n            torch.mm(eigen_vectors, torch.mm(torch.diag(eigen_vals), eigen_vectors.T))\n        )\n\n    svd_input = torch.linalg.solve_triangular(\n        triangular_mat.t(), sketch_mat, upper=True, left=False\n    )\n    left_singular_vecs, singular_vals, _ = torch.linalg.svd(\n        svd_input, full_matrices=False\n    )\n    singular_vals = torch.clamp(singular_vals**2 - shift, min=0)\n\n    return LowRankProductRepresentation(singular_vals, left_singular_vecs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.model_hessian_nystroem_approximation","title":"model_hessian_nystroem_approximation","text":"<pre><code>model_hessian_nystroem_approximation(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    data_loader: DataLoader,\n    rank: int,\n    shift_func: Optional[Callable[[Tensor], Tensor]] = None,\n) -&gt; LowRankProductRepresentation\n</code></pre> <p>Given a model, loss and a data_loader, computes a random Nystr\u00f6m low rank approximation of the corresponding Hessian matrix in factored form, i.e.</p> \\[ H_{\\text{nys}} = (H \\Omega)(\\Omega^T H \\Omega)^{+}(H \\Omega)^T = U \\Sigma U^T \\] PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model instance. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> </p> <code>data_loader</code> <p>A DataLoader instance that provides the model's training data. Used in calculating the Hessian-vector products.</p> <p> TYPE: <code>DataLoader</code> </p> <code>rank</code> <p>rank of the approximation</p> <p> TYPE: <code>int</code> </p> <code>shift_func</code> <p>optional function for computing the stabilizing shift in the construction of the randomized nystroem approximation, defaults to</p> \\[ \\sqrt{\\operatorname{\\text{input_dim}}} \\cdot     \\varepsilon(\\operatorname{\\text{input_type}}) \\cdot \\|A\\Omega\\|_2,\\] <p>where \\(\\varepsilon(\\operatorname{\\text{input_type}})\\) is the value of the machine precision corresponding to the data type.</p> <p> TYPE: <code>Optional[Callable[[Tensor], Tensor]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>object containing, \\(U\\) and \\(\\Sigma\\)</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def model_hessian_nystroem_approximation(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    data_loader: DataLoader,\n    rank: int,\n    shift_func: Optional[Callable[[torch.Tensor], torch.Tensor]] = None,\n) -&gt; LowRankProductRepresentation:\n    r\"\"\"\n    Given a model, loss and a data_loader, computes a random Nystr\u00f6m low rank approximation of\n    the corresponding Hessian matrix in factored form, i.e.\n\n    $$ H_{\\text{nys}} = (H \\Omega)(\\Omega^T H \\Omega)^{+}(H \\Omega)^T\n    = U \\Sigma U^T $$\n\n    Args:\n        model: A PyTorch model instance. The Hessian will be calculated with respect to\n            this model's parameters.\n        loss : A callable that computes the loss.\n        data_loader: A DataLoader instance that provides the model's training data.\n            Used in calculating the Hessian-vector products.\n        rank: rank of the approximation\n        shift_func: optional function for computing the stabilizing shift in the\n            construction of the randomized nystroem approximation, defaults to\n\n            $$ \\sqrt{\\operatorname{\\text{input_dim}}} \\cdot\n                \\varepsilon(\\operatorname{\\text{input_type}}) \\cdot \\|A\\Omega\\|_2,$$\n\n            where $\\varepsilon(\\operatorname{\\text{input_type}})$ is the value of the\n            machine precision corresponding to the data type.\n\n    Returns:\n        object containing, $U$ and $\\Sigma$\n    \"\"\"\n\n    model_hvp = create_hvp_function(\n        model, loss, data_loader, precompute_grad=False, use_average=True\n    )\n    device = next((p.device for p in model.parameters()))\n    dtype = next((p.dtype for p in model.parameters()))\n    in_dim = sum((p.numel() for p in model.parameters() if p.requires_grad))\n\n    def model_hessian_mat_mat_prod(x: torch.Tensor):\n        return torch.func.vmap(model_hvp, in_dims=1, randomness=\"same\")(x).t()\n\n    return randomized_nystroem_approximation(\n        model_hessian_mat_mat_prod,\n        in_dim,\n        rank,\n        dtype,\n        shift_func=shift_func,\n        mat_vec_device=device,\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/","title":"Influence function model","text":""},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model","title":"pydvl.influence.torch.influence_function_model","text":"<p>This module implements several implementations of InfluenceFunctionModel utilizing PyTorch.</p>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel","title":"TorchInfluenceFunctionModel","text":"<pre><code>TorchInfluenceFunctionModel(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor]\n)\n</code></pre> <p>             Bases: <code>InfluenceFunctionModel[Tensor, DataLoader]</code>, <code>ABC</code></p> <p>Abstract base class for influence computation related to torch models</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n):\n    self.loss = loss\n    self.model = model\n    self._n_parameters = sum(\n        [p.numel() for p in model.parameters() if p.requires_grad]\n    )\n    self._model_device = next(\n        (p.device for p in model.parameters() if p.requires_grad)\n    )\n    self._model_params = {\n        k: p.detach() for k, p in self.model.named_parameters() if p.requires_grad\n    }\n    self._model_dtype = next(\n        (p.dtype for p in model.parameters() if p.requires_grad)\n    )\n    super().__init__()\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel.is_fitted","title":"is_fitted  <code>abstractmethod</code> <code>property</code>","text":"<pre><code>is_fitted\n</code></pre> <p>Override this, to expose the fitting status of the instance.</p>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel.fit","title":"fit  <code>abstractmethod</code>","text":"<pre><code>fit(data: DataLoaderType) -&gt; InfluenceFunctionModel\n</code></pre> <p>Override this method to fit the influence function model to training data, e.g. pre-compute hessian matrix or matrix decompositions</p> PARAMETER  DESCRIPTION <code>data</code> <p> TYPE: <code>DataLoaderType</code> </p> RETURNS DESCRIPTION <code>InfluenceFunctionModel</code> <p>The fitted instance</p> Source code in <code>src/pydvl/influence/base_influence_function_model.py</code> <pre><code>@abstractmethod\ndef fit(self, data: DataLoaderType) -&gt; InfluenceFunctionModel:\n    \"\"\"\n    Override this method to fit the influence function model to training data,\n    e.g. pre-compute hessian matrix or matrix decompositions\n\n    Args:\n        data:\n\n    Returns:\n        The fitted instance\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute the approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute the approximation of\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle\n    \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle\n    \\]\n\n    for the perturbation type influence case. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x_test: model input to use in the gradient computations\n            of $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    t: torch.Tensor = super().influences(x_test, y_test, x, y, mode=mode)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.DirectInfluence","title":"DirectInfluence","text":"<pre><code>DirectInfluence(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    hessian_regularization: float = 0.0,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Given a model and training data, it finds x such that \\(Hx = b\\), with \\(H\\) being the model hessian.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns   the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>hessian_regularization</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    hessian_regularization: float = 0.0,\n):\n    super().__init__(model, loss)\n    self.hessian_regularization = hessian_regularization\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.DirectInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.DirectInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.DirectInfluence.fit","title":"fit","text":"<pre><code>fit(data: DataLoader) -&gt; DirectInfluence\n</code></pre> <p>Compute the hessian matrix based on a provided dataloader.</p> PARAMETER  DESCRIPTION <code>data</code> <p>The data to compute the Hessian with.</p> <p> TYPE: <code>DataLoader</code> </p> RETURNS DESCRIPTION <code>DirectInfluence</code> <p>The fitted instance.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def fit(self, data: DataLoader) -&gt; DirectInfluence:\n    \"\"\"\n    Compute the hessian matrix based on a provided dataloader.\n\n    Args:\n        data: The data to compute the Hessian with.\n\n    Returns:\n        The fitted instance.\n    \"\"\"\n    self.hessian = hessian(self.model, self.loss, data)\n    return self\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.DirectInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}})),     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle, \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The action of \\(H^{-1}\\) is achieved via a direct solver using torch.linalg.solve.</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A tensor representing the element-wise scalar products for the provided batch.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>@log_duration\ndef influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n        f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle, \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n        f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The action of $H^{-1}$ is achieved\n    via a direct solver using [torch.linalg.solve][torch.linalg.solve].\n\n    Args:\n        x_test: model input to use in the gradient computations of\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        A tensor representing the element-wise scalar products for the\n            provided batch.\n\n    \"\"\"\n    return super().influences(x_test, y_test, x, y, mode=mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.CgInfluence","title":"CgInfluence","text":"<pre><code>CgInfluence(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    hessian_regularization: float = 0.0,\n    x0: Optional[Tensor] = None,\n    rtol: float = 1e-07,\n    atol: float = 1e-07,\n    maxiter: Optional[int] = None,\n    progress: bool = False,\n    precompute_grad: bool = False,\n    pre_conditioner: Optional[PreConditioner] = None,\n    use_block_cg: bool = False,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Given a model and training data, it uses conjugate gradient to calculate the inverse of the Hessian Vector Product. More precisely, it finds x such that \\(Hx = b\\), with \\(H\\) being the model hessian. For more info, see Conjugate Gradient.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns   the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>hessian_regularization</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>x0</code> <p>Initial guess for hvp. If None, defaults to b.</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>rtol</code> <p>Maximum relative tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>atol</code> <p>Absolute tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>maxiter</code> <p>Maximum number of iterations. If None, defaults to 10*len(b).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>precompute_grad</code> <p>If True, the full data gradient is precomputed and kept in memory, which can speed up the hessian vector product computation. Set this to False, if you can't afford to keep the full computation graph in memory.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>pre_conditioner</code> <p>Optional pre-conditioner to improve convergence of conjugate gradient method</p> <p> TYPE: <code>Optional[PreConditioner]</code> DEFAULT: <code>None</code> </p> <code>use_block_cg</code> <p>If True, use block variant of conjugate gradient method, which solves several right hand sides simultaneously</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    hessian_regularization: float = 0.0,\n    x0: Optional[torch.Tensor] = None,\n    rtol: float = 1e-7,\n    atol: float = 1e-7,\n    maxiter: Optional[int] = None,\n    progress: bool = False,\n    precompute_grad: bool = False,\n    pre_conditioner: Optional[PreConditioner] = None,\n    use_block_cg: bool = False,\n):\n    super().__init__(model, loss)\n    self.use_block_cg = use_block_cg\n    self.pre_conditioner = pre_conditioner\n    self.precompute_grad = precompute_grad\n    self.progress = progress\n    self.maxiter = maxiter\n    self.atol = atol\n    self.rtol = rtol\n    self.x0 = x0\n    self.hessian_regularization = hessian_regularization\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.CgInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.CgInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.CgInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute an approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}})),     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle, \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of perturbation-type influence. The approximate action of \\(H^{-1}\\) is achieved via the conjugate gradient method.</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A tensor representing the element-wise scalar products for the provided batch.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>@log_duration\ndef influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute an approximation of\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n        f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle, \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n        f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of perturbation-type influence. The approximate action of\n    $H^{-1}$ is achieved via the [conjugate gradient\n    method](https://en.wikipedia.org/wiki/Conjugate_gradient_method).\n\n    Args:\n        x_test: model input to use in the gradient computations of\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        A tensor representing the element-wise scalar products for the\n            provided batch.\n\n    \"\"\"\n    return super().influences(x_test, y_test, x, y, mode=mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.LissaInfluence","title":"LissaInfluence","text":"<pre><code>LissaInfluence(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    hessian_regularization: float = 0.0,\n    maxiter: int = 1000,\n    dampen: float = 0.0,\n    scale: float = 10.0,\n    h0: Optional[Tensor] = None,\n    rtol: float = 0.0001,\n    progress: bool = False,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Uses LISSA, Linear time Stochastic Second-Order Algorithm, to iteratively approximate the inverse Hessian. More precisely, it finds x s.t. \\(Hx = b\\), with \\(H\\) being the model's second derivative wrt. the parameters. This is done with the update</p> \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\] <p>where \\(I\\) is the identity matrix, \\(d\\) is a dampening term and \\(s\\) a scaling factor that are applied to help convergence. For details, see Linear time Stochastic Second-Order Approximation (LiSSA)</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns   the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>hessian_regularization</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>maxiter</code> <p>Maximum number of iterations.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1000</code> </p> <code>dampen</code> <p>Dampening factor, defaults to 0 for no dampening.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>scale</code> <p>Scaling factor, defaults to 10.</p> <p> TYPE: <code>float</code> DEFAULT: <code>10.0</code> </p> <code>h0</code> <p>Initial guess for hvp.</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>rtol</code> <p>tolerance to use for early stopping</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0001</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    hessian_regularization: float = 0.0,\n    maxiter: int = 1000,\n    dampen: float = 0.0,\n    scale: float = 10.0,\n    h0: Optional[torch.Tensor] = None,\n    rtol: float = 1e-4,\n    progress: bool = False,\n):\n    super().__init__(model, loss)\n    self.maxiter = maxiter\n    self.hessian_regularization = hessian_regularization\n    self.progress = progress\n    self.rtol = rtol\n    self.h0 = h0\n    self.scale = scale\n    self.dampen = dampen\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.LissaInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.LissaInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute the approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute the approximation of\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle\n    \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle\n    \\]\n\n    for the perturbation type influence case. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x_test: model input to use in the gradient computations\n            of $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    t: torch.Tensor = super().influences(x_test, y_test, x, y, mode=mode)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.LissaInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.ArnoldiInfluence","title":"ArnoldiInfluence","text":"<pre><code>ArnoldiInfluence(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    hessian_regularization: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-06,\n    max_iter: Optional[int] = None,\n    eigen_computation_on_gpu: bool = False,\n    precompute_grad: bool = False,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Solves the linear system Hx = b, where H is the Hessian of the model's loss function and b is the given right-hand side vector. It employs the [implicitly restarted Arnoldi method] (https://en.wikipedia.org/wiki/Arnoldi_iteration) for computing a partial eigen decomposition, which is used fo the inversion i.e.</p> \\[x = V D^{-1} V^T b\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors. For more information, see Arnoldi.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns   the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>hessian_regularization</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. Defaults to min(model's number of parameters, max(2 times rank_estimate + 1, 20)).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm. Ignored if <code>low_rank_representation</code> is provided.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. Ignored if <code>low_rank_representation</code> is provided.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the model's device via a cupy implementation. Ensure the model size or rank_estimate is appropriate for device memory. If False, the eigen pair approximation is executed on the CPU by the scipy wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>precompute_grad</code> <p>If True, the full data gradient is precomputed and kept in memory, which can speed up the hessian vector product computation. Set this to False, if you can't afford to keep the full computation graph in memory.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    hessian_regularization: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-6,\n    max_iter: Optional[int] = None,\n    eigen_computation_on_gpu: bool = False,\n    precompute_grad: bool = False,\n):\n    super().__init__(model, loss)\n    self.hessian_regularization = hessian_regularization\n    self.rank_estimate = rank_estimate\n    self.tol = tol\n    self.max_iter = max_iter\n    self.krylov_dimension = krylov_dimension\n    self.eigen_computation_on_gpu = eigen_computation_on_gpu\n    self.precompute_grad = precompute_grad\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.ArnoldiInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.ArnoldiInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute the approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute the approximation of\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle\n    \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle\n    \\]\n\n    for the perturbation type influence case. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x_test: model input to use in the gradient computations\n            of $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    t: torch.Tensor = super().influences(x_test, y_test, x, y, mode=mode)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.ArnoldiInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.ArnoldiInfluence.fit","title":"fit","text":"<pre><code>fit(data: DataLoader) -&gt; ArnoldiInfluence\n</code></pre> <p>Fitting corresponds to the computation of the low rank decomposition</p> \\[ V D^{-1} V^T \\] <p>of the Hessian defined by the provided data loader.</p> PARAMETER  DESCRIPTION <code>data</code> <p>The data to compute the Hessian with.</p> <p> TYPE: <code>DataLoader</code> </p> RETURNS DESCRIPTION <code>ArnoldiInfluence</code> <p>The fitted instance.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def fit(self, data: DataLoader) -&gt; ArnoldiInfluence:\n    r\"\"\"\n    Fitting corresponds to the computation of the low rank decomposition\n\n    \\[ V D^{-1} V^T \\]\n\n    of the Hessian defined by the provided data loader.\n\n    Args:\n        data: The data to compute the Hessian with.\n\n    Returns:\n        The fitted instance.\n\n    \"\"\"\n    low_rank_representation = model_hessian_low_rank(\n        self.model,\n        self.loss,\n        data,\n        hessian_perturbation=0.0,  # regularization is applied, when computing values\n        rank_estimate=self.rank_estimate,\n        krylov_dimension=self.krylov_dimension,\n        tol=self.tol,\n        max_iter=self.max_iter,\n        eigen_computation_on_gpu=self.eigen_computation_on_gpu,\n        precompute_grad=self.precompute_grad,\n    )\n    self.low_rank_representation = low_rank_representation.to(self.model_device)\n    return self\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence","title":"EkfacInfluence","text":"<pre><code>EkfacInfluence(\n    model: Module,\n    update_diagonal: bool = False,\n    hessian_regularization: float = 0.0,\n    progress: bool = False,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Approximately solves the linear system Hx = b, where H is the Hessian of a model with the empirical categorical cross entropy as loss function and b is the given right-hand side vector. It employs the EK-FAC method, which is based on the kronecker factorization of the Hessian.</p> <p>Contrary to the other influence function methods, this implementation can only be used for classification tasks with a cross entropy loss function. However, it is much faster than the other methods and can be used efficiently for very large datasets and models. For more information, see Eigenvalue Corrected K-FAC.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>update_diagonal</code> <p>If True, the diagonal values in the ekfac representation are refitted from the training data after calculating the KFAC blocks. This provides a more accurate approximation of the Hessian, but it is computationally more expensive.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>hessian_regularization</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    update_diagonal: bool = False,\n    hessian_regularization: float = 0.0,\n    progress: bool = False,\n):\n    super().__init__(model, torch.nn.functional.cross_entropy)\n    self.hessian_regularization = hessian_regularization\n    self.update_diagonal = update_diagonal\n    self.active_layers = self._parse_active_layers()\n    self.progress = progress\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute the approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute the approximation of\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle\n    \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle\n    \\]\n\n    for the perturbation type influence case. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x_test: model input to use in the gradient computations\n            of $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    t: torch.Tensor = super().influences(x_test, y_test, x, y, mode=mode)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.fit","title":"fit","text":"<pre><code>fit(data: DataLoader) -&gt; EkfacInfluence\n</code></pre> <p>Compute the KFAC blocks for each layer of the model, using the provided data. It then creates an EkfacRepresentation object that stores the KFAC blocks for each layer, their eigenvalue decomposition and diagonal values.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def fit(self, data: DataLoader) -&gt; EkfacInfluence:\n    \"\"\"\n    Compute the KFAC blocks for each layer of the model, using the provided data.\n    It then creates an EkfacRepresentation object that stores the KFAC blocks for\n    each layer, their eigenvalue decomposition and diagonal values.\n    \"\"\"\n    forward_x, grad_y = self._get_kfac_blocks(data)\n    layers_evecs_a = {}\n    layers_evect_g = {}\n    layers_diags = {}\n    for key in self.active_layers.keys():\n        evals_a, evecs_a = torch.linalg.eigh(forward_x[key])\n        evals_g, evecs_g = torch.linalg.eigh(grad_y[key])\n        layers_evecs_a[key] = evecs_a\n        layers_evect_g[key] = evecs_g\n        layers_diags[key] = torch.kron(evals_g.view(-1, 1), evals_a.view(-1, 1))\n\n    self.ekfac_representation = EkfacRepresentation(\n        self.active_layers.keys(),\n        self.active_layers.values(),\n        layers_evecs_a.values(),\n        layers_evect_g.values(),\n        layers_diags.values(),\n    )\n    if self.update_diagonal:\n        self._update_diag(data)\n    return self\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influences_by_layer","title":"influences_by_layer","text":"<pre><code>influences_by_layer(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Dict[str, Tensor]\n</code></pre> <p>Compute the influence of the data on the test data for each layer of the model.</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>A dictionary containing the influence of the data on the test data for each</p> <code>Dict[str, Tensor]</code> <p>layer of the model, with the layer name as key.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_by_layer(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Dict[str, torch.Tensor]:\n    r\"\"\"\n    Compute the influence of the data on the test data for each layer of the model.\n\n    Args:\n        x_test: model input to use in the gradient computations of\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        A dictionary containing the influence of the data on the test data for each\n        layer of the model, with the layer name as key.\n    \"\"\"\n    if not self.is_fitted:\n        raise ValueError(\n            \"Instance must be fitted before calling influence methods on it\"\n        )\n\n    if x is None:\n        if y is not None:\n            raise ValueError(\n                \"Providing labels y, without providing model input x \"\n                \"is not supported\"\n            )\n\n        return self._symmetric_values_by_layer(\n            x_test.to(self.model_device),\n            y_test.to(self.model_device),\n            mode,\n        )\n\n    if y is None:\n        raise ValueError(\n            \"Providing model input x without providing labels y is not supported\"\n        )\n\n    return self._non_symmetric_values_by_layer(\n        x_test.to(self.model_device),\n        y_test.to(self.model_device),\n        x.to(self.model_device),\n        y.to(self.model_device),\n        mode,\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influence_factors_by_layer","title":"influence_factors_by_layer","text":"<pre><code>influence_factors_by_layer(x: Tensor, y: Tensor) -&gt; Dict[str, Tensor]\n</code></pre> <p>Computes the approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>for each layer of the model separately.</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>A dictionary containing the influence factors for each layer of the model,</p> <code>Dict[str, Tensor]</code> <p>with the layer name as key.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors_by_layer(\n    self,\n    x: torch.Tensor,\n    y: torch.Tensor,\n) -&gt; Dict[str, torch.Tensor]:\n    r\"\"\"\n    Computes the approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    for each layer of the model separately.\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        A dictionary containing the influence factors for each layer of the model,\n        with the layer name as key.\n    \"\"\"\n    if not self.is_fitted:\n        raise ValueError(\n            \"Instance must be fitted before calling influence methods on it\"\n        )\n\n    return self._solve_hvp_by_layer(\n        self._loss_grad(x.to(self.model_device), y.to(self.model_device)),\n        self.ekfac_representation,\n        self.hessian_regularization,\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influences_from_factors_by_layer","title":"influences_from_factors_by_layer","text":"<pre><code>influences_from_factors_by_layer(\n    z_test_factors: Dict[str, Tensor],\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Dict[str, Tensor]\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case for each layer of the model separately. The gradients are meant to be per sample of the batch \\((x, y)\\).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Dict[str, Tensor]</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>A dictionary containing the influence of the data on the test data</p> <code>Dict[str, Tensor]</code> <p>for each layer of the model, with the layer name as key.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors_by_layer(\n    self,\n    z_test_factors: Dict[str, torch.Tensor],\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Dict[str, torch.Tensor]:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case for each layer of the model\n    separately. The gradients are meant to be per sample of the batch $(x,\n    y)$.\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        A dictionary containing the influence of the data on the test data\n        for each layer of the model, with the layer name as key.\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        total_grad = self._loss_grad(\n            x.to(self.model_device), y.to(self.model_device)\n        )\n        start_idx = 0\n        influences = {}\n        for layer_id, layer_z_test in z_test_factors.items():\n            end_idx = start_idx + layer_z_test.shape[1]\n            influences[layer_id] = layer_z_test @ total_grad[:, start_idx:end_idx].T\n            start_idx = end_idx\n        return influences\n    elif mode == InfluenceMode.Perturbation:\n        total_mixed_grad = self._flat_loss_mixed_grad(\n            x.to(self.model_device), y.to(self.model_device)\n        )\n        start_idx = 0\n        influences = {}\n        for layer_id, layer_z_test in z_test_factors.items():\n            end_idx = start_idx + layer_z_test.shape[1]\n            influences[layer_id] = torch.einsum(\n                \"ia,j...a-&gt;ij...\",\n                layer_z_test,\n                total_mixed_grad[:, start_idx:end_idx],\n            )\n            start_idx = end_idx\n        return influences\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.explore_hessian_regularization","title":"explore_hessian_regularization","text":"<pre><code>explore_hessian_regularization(\n    x: Tensor, y: Tensor, regularization_values: List[float]\n) -&gt; Dict[float, Dict[str, Tensor]]\n</code></pre> <p>Efficiently computes the influence for input x and label y for each layer of the model, for different values of the hessian regularization parameter. This is done by computing the gradient of the loss function for the input x and label y only once and then solving the Hessian Vector Product for each regularization value. This is useful for finding the optimal regularization value and for exploring how robust the influence values are to changes in the regularization value.</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>regularization_values</code> <p>list of regularization values to use</p> <p> TYPE: <code>List[float]</code> </p> RETURNS DESCRIPTION <code>Dict[float, Dict[str, Tensor]]</code> <p>A dictionary containing with keys being the regularization values and values</p> <code>Dict[float, Dict[str, Tensor]]</code> <p>being dictionaries containing the influences for each layer of the model,</p> <code>Dict[float, Dict[str, Tensor]]</code> <p>with the layer name as key.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def explore_hessian_regularization(\n    self,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    regularization_values: List[float],\n) -&gt; Dict[float, Dict[str, torch.Tensor]]:\n    \"\"\"\n    Efficiently computes the influence for input x and label y for each layer of the\n    model, for different values of the hessian regularization parameter. This is done\n    by computing the gradient of the loss function for the input x and label y only once\n    and then solving the Hessian Vector Product for each regularization value. This is\n    useful for finding the optimal regularization value and for exploring\n    how robust the influence values are to changes in the regularization value.\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n        regularization_values: list of regularization values to use\n\n    Returns:\n        A dictionary containing with keys being the regularization values and values\n        being dictionaries containing the influences for each layer of the model,\n        with the layer name as key.\n    \"\"\"\n    grad = self._loss_grad(x, y)\n    influences_by_reg_value = {}\n    for reg_value in regularization_values:\n        reg_factors = self._solve_hvp_by_layer(\n            grad, self.ekfac_representation, reg_value\n        )\n        values = {}\n        start_idx = 0\n        for layer_id, layer_fac in reg_factors.items():\n            end_idx = start_idx + layer_fac.shape[1]\n            values[layer_id] = layer_fac @ grad[:, start_idx:end_idx].T\n            start_idx = end_idx\n        influences_by_reg_value[reg_value] = values\n    return influences_by_reg_value\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.NystroemSketchInfluence","title":"NystroemSketchInfluence","text":"<pre><code>NystroemSketchInfluence(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    hessian_regularization: float,\n    rank: int,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Given a model and training data, it uses a low-rank approximation of the Hessian (derived via random projection Nystr\u00f6m approximation) in combination with the Sherman\u2013Morrison\u2013Woodbury formula to calculate the inverse of the Hessian Vector Product. More concrete, it computes a low-rank approximation</p> \\[\\begin{align*}     H_{\\text{nys}} &amp;= (H\\Omega)(\\Omega^TH\\Omega)^{+}(H\\Omega)^T \\\\\\                    &amp;= U \\Lambda U^T \\end{align*}\\] <p>in factorized form and approximates the action of the inverse Hessian via</p> \\[ (H_{\\text{nys}} + \\lambda I)^{-1} = U(\\Lambda+\\lambda I)U^T +     \\frac{1}{\\lambda}(I\u2212UU^T). \\] PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns   the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>hessian_regularization</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> </p> <code>rank</code> <p>rank of the low-rank approximation</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    hessian_regularization: float,\n    rank: int,\n):\n    super().__init__(model, loss)\n    self.hessian_regularization = hessian_regularization\n    self.rank = rank\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.NystroemSketchInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.NystroemSketchInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute the approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute the approximation of\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle\n    \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle\n    \\]\n\n    for the perturbation type influence case. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x_test: model input to use in the gradient computations\n            of $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    t: torch.Tensor = super().influences(x_test, y_test, x, y, mode=mode)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.NystroemSketchInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/","title":"Pre conditioner","text":""},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner","title":"pydvl.influence.torch.pre_conditioner","text":""},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.PreConditioner","title":"PreConditioner","text":"<p>             Bases: <code>ABC</code></p> <p>Abstract base class for implementing pre-conditioners for improving the convergence of CG for systems of the form</p> \\[ ( A + \\lambda \\operatorname{I})x = \\operatorname{rhs} \\] <p>i.e. a matrix \\(M\\) such that \\(M^{-1}(A + \\lambda \\operatorname{I})\\) has a better condition number than \\(A + \\lambda \\operatorname{I}\\).</p>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.PreConditioner.fit","title":"fit  <code>abstractmethod</code>","text":"<pre><code>fit(\n    mat_mat_prod: Callable[[Tensor], Tensor],\n    size: int,\n    dtype: dtype,\n    device: device,\n    regularization: float = 0.0,\n)\n</code></pre> <p>Implement this to fit the pre-conditioner to the matrix represented by the mat_mat_prod Args:     mat_mat_prod: a callable that computes the matrix-matrix product     size: size of the matrix represented by <code>mat_mat_prod</code>     dtype: data type of the matrix represented by <code>mat_mat_prod</code>     device: device of the matrix represented by <code>mat_mat_prod</code>     regularization: regularization parameter \\(\\lambda\\) in the equation         $ ( A + \\lambda \\operatorname{I})x = \\operatorname{rhs} $ Returns:     self</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>@abstractmethod\ndef fit(\n    self,\n    mat_mat_prod: Callable[[torch.Tensor], torch.Tensor],\n    size: int,\n    dtype: torch.dtype,\n    device: torch.device,\n    regularization: float = 0.0,\n):\n    r\"\"\"\n    Implement this to fit the pre-conditioner to the matrix represented by the\n    mat_mat_prod\n    Args:\n        mat_mat_prod: a callable that computes the matrix-matrix product\n        size: size of the matrix represented by `mat_mat_prod`\n        dtype: data type of the matrix represented by `mat_mat_prod`\n        device: device of the matrix represented by `mat_mat_prod`\n        regularization: regularization parameter $\\lambda$ in the equation\n            $ ( A + \\lambda \\operatorname{I})x = \\operatorname{rhs} $\n    Returns:\n        self\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.PreConditioner.solve","title":"solve","text":"<pre><code>solve(rhs: Tensor)\n</code></pre> <p>Solve the equation \\(M@Z = \\operatorname{rhs}\\) Args:     rhs: right hand side of the equation, corresponds to the residuum vector         (or matrix) in the conjugate gradient method</p> RETURNS DESCRIPTION <p>solution \\(M^{-1}\\operatorname{rhs}\\)</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def solve(self, rhs: torch.Tensor):\n    r\"\"\"\n    Solve the equation $M@Z = \\operatorname{rhs}$\n    Args:\n        rhs: right hand side of the equation, corresponds to the residuum vector\n            (or matrix) in the conjugate gradient method\n\n    Returns:\n        solution $M^{-1}\\operatorname{rhs}$\n\n    \"\"\"\n    if not self.is_fitted:\n        raise NotFittedException(type(self))\n\n    return self._solve(rhs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.JacobiPreConditioner","title":"JacobiPreConditioner","text":"<pre><code>JacobiPreConditioner(num_samples_estimator: int = 1)\n</code></pre> <p>             Bases: <code>PreConditioner</code></p> <p>Pre-conditioner for improving the convergence of CG for systems of the form</p> \\[ ( A + \\lambda \\operatorname{I})x = \\operatorname{rhs} \\] <p>The JacobiPreConditioner uses the diagonal information of the matrix \\(A\\). The diagonal elements are not computed directly but estimated via Hutchinson's estimator.</p> \\[ M = \\frac{1}{m} \\sum_{i=1}^m u_i \\odot Au_i + \\lambda \\operatorname{I} \\] <p>where \\(u_i\\) are i.i.d. Gaussian random vectors. Works well in the case the matrix \\(A + \\lambda \\operatorname{I}\\) is diagonal dominant. For more information, see the documentation of Conjugate Gradient Args:     num_samples_estimator: number of samples to use in computation of         Hutchinson's estimator</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def __init__(self, num_samples_estimator: int = 1):\n    self.num_samples_estimator = num_samples_estimator\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.JacobiPreConditioner.solve","title":"solve","text":"<pre><code>solve(rhs: Tensor)\n</code></pre> <p>Solve the equation \\(M@Z = \\operatorname{rhs}\\) Args:     rhs: right hand side of the equation, corresponds to the residuum vector         (or matrix) in the conjugate gradient method</p> RETURNS DESCRIPTION <p>solution \\(M^{-1}\\operatorname{rhs}\\)</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def solve(self, rhs: torch.Tensor):\n    r\"\"\"\n    Solve the equation $M@Z = \\operatorname{rhs}$\n    Args:\n        rhs: right hand side of the equation, corresponds to the residuum vector\n            (or matrix) in the conjugate gradient method\n\n    Returns:\n        solution $M^{-1}\\operatorname{rhs}$\n\n    \"\"\"\n    if not self.is_fitted:\n        raise NotFittedException(type(self))\n\n    return self._solve(rhs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.JacobiPreConditioner.fit","title":"fit","text":"<pre><code>fit(\n    mat_mat_prod: Callable[[Tensor], Tensor],\n    size: int,\n    dtype: dtype,\n    device: device,\n    regularization: float = 0.0,\n)\n</code></pre> <p>Fits by computing an estimate of the diagonal of the matrix represented by <code>mat_mat_prod</code> via Hutchinson's estimator</p> PARAMETER  DESCRIPTION <code>mat_mat_prod</code> <p>a callable representing the matrix-matrix product</p> <p> TYPE: <code>Callable[[Tensor], Tensor]</code> </p> <code>size</code> <p>size of the square matrix</p> <p> TYPE: <code>int</code> </p> <code>dtype</code> <p>needed data type of inputs for the mat_mat_prod</p> <p> TYPE: <code>dtype</code> </p> <code>device</code> <p>needed device for inputs of mat_mat_prod</p> <p> TYPE: <code>device</code> </p> <code>regularization</code> <p>regularization parameter \\(\\lambda\\) in \\((A+\\lambda I)x=b\\)</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def fit(\n    self,\n    mat_mat_prod: Callable[[torch.Tensor], torch.Tensor],\n    size: int,\n    dtype: torch.dtype,\n    device: torch.device,\n    regularization: float = 0.0,\n):\n    r\"\"\"\n    Fits by computing an estimate of the diagonal of the matrix represented by\n    `mat_mat_prod` via Hutchinson's estimator\n\n    Args:\n        mat_mat_prod: a callable representing the matrix-matrix product\n        size: size of the square matrix\n        dtype: needed data type of inputs for the mat_mat_prod\n        device: needed device for inputs of mat_mat_prod\n        regularization: regularization parameter\n            $\\lambda$ in $(A+\\lambda I)x=b$\n    \"\"\"\n    random_samples = torch.randn(\n        size, self.num_samples_estimator, device=device, dtype=dtype\n    )\n    diagonal_estimate = torch.sum(\n        torch.mul(random_samples, mat_mat_prod(random_samples)), dim=1\n    )\n    diagonal_estimate /= self.num_samples_estimator\n    self._diag = diagonal_estimate\n    self._reg = regularization\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.NystroemPreConditioner","title":"NystroemPreConditioner","text":"<pre><code>NystroemPreConditioner(rank: int)\n</code></pre> <p>             Bases: <code>PreConditioner</code></p> <p>Pre-conditioner for improving the convergence of CG for systems of the form</p> \\[ (A + \\lambda \\operatorname{I})x = \\operatorname{rhs} \\] <p>The NystroemPreConditioner computes a low-rank approximation</p> \\[ A_{\\text{nys}} = (A \\Omega)(\\Omega^T A \\Omega)^{\\dagger}(A \\Omega)^T = U \\Sigma U^T, \\] <p>where \\((\\cdot)^{\\dagger}\\) denotes the Moore-Penrose inverse, and uses the matrix</p> \\[ M^{-1} = (\\lambda + \\sigma_{\\text{rank}})U(\\Sigma+     \\lambda \\operatorname{I})^{-1}U^T+(\\operatorname{I} - UU^T) \\] <p>for pre-conditioning, where \\(  \\sigma_{\\text{rank}} \\) is the smallest eigenvalue of the low-rank approximation.</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def __init__(self, rank: int):\n    self._rank = rank\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.NystroemPreConditioner.solve","title":"solve","text":"<pre><code>solve(rhs: Tensor)\n</code></pre> <p>Solve the equation \\(M@Z = \\operatorname{rhs}\\) Args:     rhs: right hand side of the equation, corresponds to the residuum vector         (or matrix) in the conjugate gradient method</p> RETURNS DESCRIPTION <p>solution \\(M^{-1}\\operatorname{rhs}\\)</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def solve(self, rhs: torch.Tensor):\n    r\"\"\"\n    Solve the equation $M@Z = \\operatorname{rhs}$\n    Args:\n        rhs: right hand side of the equation, corresponds to the residuum vector\n            (or matrix) in the conjugate gradient method\n\n    Returns:\n        solution $M^{-1}\\operatorname{rhs}$\n\n    \"\"\"\n    if not self.is_fitted:\n        raise NotFittedException(type(self))\n\n    return self._solve(rhs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.NystroemPreConditioner.fit","title":"fit","text":"<pre><code>fit(\n    mat_mat_prod: Callable[[Tensor], Tensor],\n    size: int,\n    dtype: dtype,\n    device: device,\n    regularization: float = 0.0,\n)\n</code></pre> <p>Fits by computing a low-rank approximation of the matrix represented by <code>mat_mat_prod</code> via Nystroem approximation</p> PARAMETER  DESCRIPTION <code>mat_mat_prod</code> <p>a callable representing the matrix-matrix product</p> <p> TYPE: <code>Callable[[Tensor], Tensor]</code> </p> <code>size</code> <p>size of the square matrix</p> <p> TYPE: <code>int</code> </p> <code>dtype</code> <p>needed data type of inputs for the mat_mat_prod</p> <p> TYPE: <code>dtype</code> </p> <code>device</code> <p>needed device for inputs of mat_mat_prod</p> <p> TYPE: <code>device</code> </p> <code>regularization</code> <p>regularization parameter \\(\\lambda\\)  in \\((A+\\lambda I)x=b\\)</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def fit(\n    self,\n    mat_mat_prod: Callable[[torch.Tensor], torch.Tensor],\n    size: int,\n    dtype: torch.dtype,\n    device: torch.device,\n    regularization: float = 0.0,\n):\n    r\"\"\"\n    Fits by computing a low-rank approximation of the matrix represented by\n    `mat_mat_prod` via Nystroem approximation\n\n    Args:\n        mat_mat_prod: a callable representing the matrix-matrix product\n        size: size of the square matrix\n        dtype: needed data type of inputs for the mat_mat_prod\n        device: needed device for inputs of mat_mat_prod\n        regularization: regularization parameter\n            $\\lambda$  in $(A+\\lambda I)x=b$\n    \"\"\"\n\n    self._low_rank_approx = randomized_nystroem_approximation(\n        mat_mat_prod, size, self._rank, dtype, mat_vec_device=device\n    )\n    self._regularization = regularization\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/","title":"Util","text":""},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util","title":"pydvl.influence.torch.util","text":""},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchTensorContainerType","title":"TorchTensorContainerType  <code>module-attribute</code>","text":"<pre><code>TorchTensorContainerType = Union[\n    Tensor, Collection[Tensor], Mapping[str, Tensor]\n]\n</code></pre> <p>Type for a PyTorch tensor or a container thereof.</p>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchNumpyConverter","title":"TorchNumpyConverter","text":"<pre><code>TorchNumpyConverter(device: Optional[device] = None)\n</code></pre> <p>             Bases: <code>NumpyConverter[Tensor]</code></p> <p>Helper class for converting between torch.Tensor and numpy.ndarray</p> PARAMETER  DESCRIPTION <code>device</code> <p>Optional device parameter to move the resulting torch tensors to the specified device</p> <p> TYPE: <code>Optional[device]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def __init__(self, device: Optional[torch.device] = None):\n    self.device = device\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchNumpyConverter.to_numpy","title":"to_numpy","text":"<pre><code>to_numpy(x: Tensor) -&gt; NDArray\n</code></pre> <p>Convert a detached torch.Tensor to numpy.ndarray</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def to_numpy(self, x: torch.Tensor) -&gt; NDArray:\n    \"\"\"\n    Convert a detached [torch.Tensor][torch.Tensor] to\n    [numpy.ndarray][numpy.ndarray]\n    \"\"\"\n    arr: NDArray = x.cpu().numpy()\n    return arr\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchNumpyConverter.from_numpy","title":"from_numpy","text":"<pre><code>from_numpy(x: NDArray) -&gt; Tensor\n</code></pre> <p>Convert a numpy.ndarray to torch.Tensor and optionally move it to a provided device</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def from_numpy(self, x: NDArray) -&gt; torch.Tensor:\n    \"\"\"\n    Convert a [numpy.ndarray][numpy.ndarray] to [torch.Tensor][torch.Tensor] and\n    optionally move it to a provided device\n    \"\"\"\n    t = torch.from_numpy(x)\n    if self.device is not None:\n        t = t.to(self.device)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchCatAggregator","title":"TorchCatAggregator","text":"<p>             Bases: <code>SequenceAggregator[Tensor]</code></p> <p>An aggregator that concatenates tensors using PyTorch's torch.cat function. Concatenation is done along the first dimension of the chunks.</p>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchCatAggregator.__call__","title":"__call__","text":"<pre><code>__call__(tensor_generator: Generator[Tensor, None, None])\n</code></pre> <p>Aggregates tensors from a single-level generator into a single tensor by concatenating them. This method is a straightforward way to combine a sequence of tensors into one larger tensor.</p> PARAMETER  DESCRIPTION <code>tensor_generator</code> <p>A generator that yields <code>torch.Tensor</code> objects.</p> <p> TYPE: <code>Generator[Tensor, None, None]</code> </p> RETURNS DESCRIPTION <p>A single tensor formed by concatenating all tensors from the generator. The concatenation is performed along the default dimension (0).</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def __call__(self, tensor_generator: Generator[torch.Tensor, None, None]):\n    \"\"\"\n    Aggregates tensors from a single-level generator into a single tensor by\n    concatenating them. This method is a straightforward way to combine a sequence\n    of tensors into one larger tensor.\n\n    Args:\n        tensor_generator: A generator that yields `torch.Tensor` objects.\n\n    Returns:\n        A single tensor formed by concatenating all tensors from the generator.\n            The concatenation is performed along the default dimension (0).\n    \"\"\"\n    return torch.cat(list(tensor_generator))\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.NestedTorchCatAggregator","title":"NestedTorchCatAggregator","text":"<p>             Bases: <code>NestedSequenceAggregator[Tensor]</code></p> <p>An aggregator that concatenates tensors using PyTorch's torch.cat function. Concatenation is done along the first two dimensions of the chunks.</p>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.NestedTorchCatAggregator.__call__","title":"__call__","text":"<pre><code>__call__(\n    nested_generators_of_tensors: Generator[\n        Generator[Tensor, None, None], None, None\n    ]\n)\n</code></pre> <p>Aggregates tensors from a nested generator structure into a single tensor by concatenating. Each inner generator is first concatenated along dimension 1 into a tensor, and then these tensors are concatenated along dimension 0 together to form the final tensor.</p> PARAMETER  DESCRIPTION <code>nested_generators_of_tensors</code> <p>A generator of generators, where each inner generator yields <code>torch.Tensor</code> objects.</p> <p> TYPE: <code>Generator[Generator[Tensor, None, None], None, None]</code> </p> RETURNS DESCRIPTION <p>A single tensor formed by concatenating all tensors from the nested</p> <p>generators.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def __call__(\n    self,\n    nested_generators_of_tensors: Generator[\n        Generator[torch.Tensor, None, None], None, None\n    ],\n):\n    \"\"\"\n    Aggregates tensors from a nested generator structure into a single tensor by\n    concatenating. Each inner generator is first concatenated along dimension 1 into\n    a tensor, and then these tensors are concatenated along dimension 0 together to\n    form the final tensor.\n\n    Args:\n        nested_generators_of_tensors: A generator of generators, where each inner\n            generator yields `torch.Tensor` objects.\n\n    Returns:\n        A single tensor formed by concatenating all tensors from the nested\n        generators.\n\n    \"\"\"\n    return torch.cat(\n        list(\n            map(\n                lambda tensor_gen: torch.cat(list(tensor_gen), dim=1),\n                nested_generators_of_tensors,\n            )\n        )\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.EkfacRepresentation","title":"EkfacRepresentation  <code>dataclass</code>","text":"<pre><code>EkfacRepresentation(\n    layer_names: Iterable[str],\n    layers_module: Iterable[Module],\n    evecs_a: Iterable[Tensor],\n    evecs_g: Iterable[Tensor],\n    diags: Iterable[Tensor],\n)\n</code></pre> <p>Container class for the EKFAC representation of the Hessian. It can be iterated over to get the layers names and their corresponding module, eigenvectors and diagonal elements of the factorized Hessian matrix.</p> PARAMETER  DESCRIPTION <code>layer_names</code> <p>Names of the layers.</p> <p> TYPE: <code>Iterable[str]</code> </p> <code>layers_module</code> <p>The layers.</p> <p> TYPE: <code>Iterable[Module]</code> </p> <code>evecs_a</code> <p>The a eigenvectors of the ekfac representation.</p> <p> TYPE: <code>Iterable[Tensor]</code> </p> <code>evecs_g</code> <p>The g eigenvectors of the ekfac representation.</p> <p> TYPE: <code>Iterable[Tensor]</code> </p> <code>diags</code> <p>The diagonal elements of the factorized Hessian matrix.</p> <p> TYPE: <code>Iterable[Tensor]</code> </p>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.EkfacRepresentation.get_layer_evecs","title":"get_layer_evecs","text":"<pre><code>get_layer_evecs() -&gt; Tuple[Dict[str, Tensor], Dict[str, Tensor]]\n</code></pre> <p>It returns two dictionaries, one for the a eigenvectors and one for the g eigenvectors, with the layer names as keys. The eigenvectors are in the same order as the layers in the model.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def get_layer_evecs(\n    self,\n) -&gt; Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]]:\n    \"\"\"\n    It returns two dictionaries, one for the a eigenvectors and one for the g\n    eigenvectors, with the layer names as keys. The eigenvectors are in the same\n    order as the layers in the model.\n    \"\"\"\n    evecs_a_dict = {layer_name: evec_a for layer_name, (_, evec_a, _, _) in self}\n    evecs_g_dict = {layer_name: evec_g for layer_name, (_, _, evec_g, _) in self}\n    return evecs_a_dict, evecs_g_dict\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.to_model_device","title":"to_model_device","text":"<pre><code>to_model_device(x: Tensor, model: Module) -&gt; Tensor\n</code></pre> <p>Returns the tensor <code>x</code> moved to the device of the <code>model</code>, if device of model is set</p> PARAMETER  DESCRIPTION <code>x</code> <p>The tensor to be moved to the device of the model.</p> <p> TYPE: <code>Tensor</code> </p> <code>model</code> <p>The model whose device will be used to move the tensor.</p> <p> TYPE: <code>Module</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>The tensor <code>x</code> moved to the device of the <code>model</code>, if device of model is set.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def to_model_device(x: torch.Tensor, model: torch.nn.Module) -&gt; torch.Tensor:\n    \"\"\"\n    Returns the tensor `x` moved to the device of the `model`, if device of model is set\n\n    Args:\n        x: The tensor to be moved to the device of the model.\n        model: The model whose device will be used to move the tensor.\n\n    Returns:\n        The tensor `x` moved to the device of the `model`, if device of model is set.\n    \"\"\"\n    device = next(model.parameters()).device\n    return x.to(device)\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.reshape_vector_to_tensors","title":"reshape_vector_to_tensors","text":"<pre><code>reshape_vector_to_tensors(\n    input_vector: Tensor, target_shapes: Iterable[Tuple[int, ...]]\n) -&gt; Tuple[Tensor, ...]\n</code></pre> <p>Reshape a 1D tensor into multiple tensors with specified shapes.</p> <p>This function takes a 1D tensor (input_vector) and reshapes it into a series of tensors with shapes given by 'target_shapes'. The reshaped tensors are returned as a tuple in the same order as their corresponding shapes.</p> Note <p>The total number of elements in 'input_vector' must be equal to the     sum of the products of the shapes in 'target_shapes'.</p> PARAMETER  DESCRIPTION <code>input_vector</code> <p>The 1D tensor to be reshaped. Must be 1D.</p> <p> TYPE: <code>Tensor</code> </p> <code>target_shapes</code> <p>An iterable of tuples. Each tuple defines the shape of a tensor to be reshaped from the 'input_vector'.</p> <p> TYPE: <code>Iterable[Tuple[int, ...]]</code> </p> RETURNS DESCRIPTION <code>Tuple[Tensor, ...]</code> <p>A tuple of reshaped tensors.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If 'input_vector' is not a 1D tensor or if the total number of elements in 'input_vector' does not match the sum of the products of the shapes in 'target_shapes'.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def reshape_vector_to_tensors(\n    input_vector: torch.Tensor, target_shapes: Iterable[Tuple[int, ...]]\n) -&gt; Tuple[torch.Tensor, ...]:\n    \"\"\"\n    Reshape a 1D tensor into multiple tensors with specified shapes.\n\n    This function takes a 1D tensor (input_vector) and reshapes it into a series of\n    tensors with shapes given by 'target_shapes'.\n    The reshaped tensors are returned as a tuple in the same order\n    as their corresponding shapes.\n\n    Note:\n        The total number of elements in 'input_vector' must be equal to the\n            sum of the products of the shapes in 'target_shapes'.\n\n    Args:\n        input_vector: The 1D tensor to be reshaped. Must be 1D.\n        target_shapes: An iterable of tuples. Each tuple defines the shape of a tensor\n            to be reshaped from the 'input_vector'.\n\n    Returns:\n        A tuple of reshaped tensors.\n\n    Raises:\n        ValueError: If 'input_vector' is not a 1D tensor or if the total\n            number of elements in 'input_vector' does not\n            match the sum of the products of the shapes in 'target_shapes'.\n    \"\"\"\n\n    if input_vector.dim() != 1:\n        raise ValueError(\"Input vector must be a 1D tensor\")\n\n    total_elements = sum(math.prod(shape) for shape in target_shapes)\n\n    if total_elements != input_vector.shape[0]:\n        raise ValueError(\n            f\"The total elements in shapes {total_elements} \"\n            f\"does not match the vector length {input_vector.shape[0]}\"\n        )\n\n    tensors = []\n    start = 0\n    for shape in target_shapes:\n        size = math.prod(shape)  # compute the total size of the tensor with this shape\n        tensors.append(\n            input_vector[start : start + size].view(shape)\n        )  # slice the vector and reshape it\n        start += size\n    return tuple(tensors)\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.align_structure","title":"align_structure","text":"<pre><code>align_structure(\n    source: Mapping[str, Tensor], target: TorchTensorContainerType\n) -&gt; Dict[str, Tensor]\n</code></pre> <p>This function transforms <code>target</code> to have the same structure as <code>source</code>, i.e., it should be a dictionary with the same keys as <code>source</code> and each corresponding value in <code>target</code> should have the same shape as the value in <code>source</code>.</p> PARAMETER  DESCRIPTION <code>source</code> <p>The reference dictionary containing PyTorch tensors.</p> <p> TYPE: <code>Mapping[str, Tensor]</code> </p> <code>target</code> <p>The input to be harmonized. It can be a dictionary, tuple, or tensor.</p> <p> TYPE: <code>TorchTensorContainerType</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>The harmonized version of <code>target</code>.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>target</code> cannot be harmonized to match <code>source</code>.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def align_structure(\n    source: Mapping[str, torch.Tensor],\n    target: TorchTensorContainerType,\n) -&gt; Dict[str, torch.Tensor]:\n    \"\"\"\n    This function transforms `target` to have the same structure as `source`, i.e.,\n    it should be a dictionary with the same keys as `source` and each corresponding\n    value in `target` should have the same shape as the value in `source`.\n\n    Args:\n        source: The reference dictionary containing PyTorch tensors.\n        target: The input to be harmonized. It can be a dictionary, tuple, or tensor.\n\n    Returns:\n        The harmonized version of `target`.\n\n    Raises:\n        ValueError: If `target` cannot be harmonized to match `source`.\n    \"\"\"\n\n    tangent_dict: Dict[str, torch.Tensor]\n\n    if isinstance(target, dict):\n        if list(target.keys()) != list(source.keys()):\n            raise ValueError(\"The keys in 'target' do not match the keys in 'source'.\")\n\n        if [v.shape for v in target.values()] != [v.shape for v in source.values()]:\n            raise ValueError(\n                \"The shapes of the values in 'target' do not match the shapes \"\n                \"of the values in 'source'.\"\n            )\n\n        tangent_dict = target\n\n    elif isinstance(target, tuple) or isinstance(target, list):\n        if [v.shape for v in target] != [v.shape for v in source.values()]:\n            raise ValueError(\n                \"'target' is a tuple/list but its elements' shapes do not match \"\n                \"the shapes of the values in 'source'.\"\n            )\n\n        tangent_dict = dict(zip(source.keys(), target))\n\n    elif isinstance(target, torch.Tensor):\n        try:\n            tangent_dict = dict(\n                zip(\n                    source.keys(),\n                    reshape_vector_to_tensors(\n                        target, [p.shape for p in source.values()]\n                    ),\n                )\n            )\n        except Exception as e:\n            raise ValueError(\n                f\"'target' is a tensor but cannot be reshaped to match 'source'. \"\n                f\"Original error: {e}\"\n            )\n\n    else:\n        raise ValueError(f\"'target' is of type {type(target)} which is not supported.\")\n\n    return tangent_dict\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.align_with_model","title":"align_with_model","text":"<pre><code>align_with_model(x: TorchTensorContainerType, model: Module)\n</code></pre> <p>Aligns an input to the model's parameter structure, i.e. transforms it into a dict with the same keys as model.named_parameters() and matching tensor shapes</p> PARAMETER  DESCRIPTION <code>x</code> <p>The input to be aligned. It can be a dictionary, tuple, or tensor.</p> <p> TYPE: <code>TorchTensorContainerType</code> </p> <code>model</code> <p>model to use for alignment</p> <p> TYPE: <code>Module</code> </p> RETURNS DESCRIPTION <p>The aligned version of <code>x</code>.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>x</code> cannot be aligned to match the model's parameters .</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def align_with_model(x: TorchTensorContainerType, model: torch.nn.Module):\n    \"\"\"\n    Aligns an input to the model's parameter structure, i.e. transforms it into a dict\n    with the same keys as model.named_parameters() and matching tensor shapes\n\n    Args:\n        x: The input to be aligned. It can be a dictionary, tuple, or tensor.\n        model: model to use for alignment\n\n    Returns:\n        The aligned version of `x`.\n\n    Raises:\n        ValueError: If `x` cannot be aligned to match the model's parameters .\n\n    \"\"\"\n    model_params = {k: p for k, p in model.named_parameters() if p.requires_grad}\n    return align_structure(model_params, x)\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.flatten_dimensions","title":"flatten_dimensions","text":"<pre><code>flatten_dimensions(\n    tensors: Iterable[Tensor],\n    shape: Optional[Tuple[int, ...]] = None,\n    concat_at: int = -1,\n) -&gt; Tensor\n</code></pre> <p>Flattens the dimensions of each tensor in the given iterable and concatenates them along a specified dimension.</p> <p>This function takes an iterable of PyTorch tensors and flattens each tensor. Optionally, each tensor can be reshaped to a specified shape before concatenation. The concatenation is performed along the dimension specified by <code>concat_at</code>.</p> PARAMETER  DESCRIPTION <code>tensors</code> <p>An iterable containing PyTorch tensors to be flattened and concatenated.</p> <p> TYPE: <code>Iterable[Tensor]</code> </p> <code>shape</code> <p>A tuple representing the desired shape to which each tensor is reshaped before concatenation. If None, tensors are flattened to 1D.</p> <p> TYPE: <code>Optional[Tuple[int, ...]]</code> DEFAULT: <code>None</code> </p> <code>concat_at</code> <p>The dimension along which to concatenate the tensors.</p> <p> TYPE: <code>int</code> DEFAULT: <code>-1</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A single tensor resulting from the concatenation of the input tensors,</p> <code>Tensor</code> <p>each either flattened or reshaped as specified.</p> Example <pre><code>&gt;&gt;&gt; tensors = [torch.tensor([[1, 2], [3, 4]]), torch.tensor([[5, 6], [7, 8]])]\n&gt;&gt;&gt; flatten_dimensions(tensors)\ntensor([1, 2, 3, 4, 5, 6, 7, 8])\n\n&gt;&gt;&gt; flatten_dimensions(tensors, shape=(2, 2), concat_at=0)\ntensor([[1, 2],\n        [3, 4],\n        [5, 6],\n        [7, 8]])\n</code></pre> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def flatten_dimensions(\n    tensors: Iterable[torch.Tensor],\n    shape: Optional[Tuple[int, ...]] = None,\n    concat_at: int = -1,\n) -&gt; torch.Tensor:\n    \"\"\"\n    Flattens the dimensions of each tensor in the given iterable and concatenates them\n    along a specified dimension.\n\n    This function takes an iterable of PyTorch tensors and flattens each tensor.\n    Optionally, each tensor can be reshaped to a specified shape before concatenation.\n    The concatenation is performed along the dimension specified by `concat_at`.\n\n    Args:\n        tensors: An iterable containing PyTorch tensors to be flattened\n            and concatenated.\n        shape: A tuple representing the desired shape to which each tensor is reshaped\n            before concatenation. If None, tensors are flattened to 1D.\n        concat_at: The dimension along which to concatenate the tensors.\n\n    Returns:\n        A single tensor resulting from the concatenation of the input tensors,\n        each either flattened or reshaped as specified.\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; tensors = [torch.tensor([[1, 2], [3, 4]]), torch.tensor([[5, 6], [7, 8]])]\n        &gt;&gt;&gt; flatten_dimensions(tensors)\n        tensor([1, 2, 3, 4, 5, 6, 7, 8])\n\n        &gt;&gt;&gt; flatten_dimensions(tensors, shape=(2, 2), concat_at=0)\n        tensor([[1, 2],\n                [3, 4],\n                [5, 6],\n                [7, 8]])\n        ```\n    \"\"\"\n    return torch.cat(\n        [t.reshape(-1) if shape is None else t.reshape(*shape) for t in tensors],\n        dim=concat_at,\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.torch_dataset_to_dask_array","title":"torch_dataset_to_dask_array","text":"<pre><code>torch_dataset_to_dask_array(\n    dataset: Dataset,\n    chunk_size: int,\n    total_size: Optional[int] = None,\n    resulting_dtype: Type[number] = np.float32,\n) -&gt; Tuple[Array, ...]\n</code></pre> <p>Construct tuple of dask arrays from a PyTorch dataset, using dask.delayed</p> PARAMETER  DESCRIPTION <code>dataset</code> <p>A PyTorch dataset</p> <p> TYPE: <code>Dataset</code> </p> <code>chunk_size</code> <p>The size of the chunks for the resulting Dask arrays.</p> <p> TYPE: <code>int</code> </p> <code>total_size</code> <p>If the dataset does not implement len, provide the length via this parameter. If None the length of the dataset is inferred via accessing the dataset once.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>resulting_dtype</code> <p>The dtype of the resulting dask.array.Array</p> <p> TYPE: <code>Type[number]</code> DEFAULT: <code>float32</code> </p> Example <pre><code>import torch\nfrom torch.utils.data import TensorDataset\nx = torch.rand((20, 3))\ny = torch.rand((20, 1))\ndataset = TensorDataset(x, y)\nda_x, da_y = torch_dataset_to_dask_array(dataset, 4)\n</code></pre> RETURNS DESCRIPTION <code>Tuple[Array, ...]</code> <p>Tuple of Dask arrays corresponding to each tensor in the dataset.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def torch_dataset_to_dask_array(\n    dataset: Dataset,\n    chunk_size: int,\n    total_size: Optional[int] = None,\n    resulting_dtype: Type[np.number] = np.float32,\n) -&gt; Tuple[da.Array, ...]:\n    \"\"\"\n    Construct tuple of dask arrays from a PyTorch dataset, using dask.delayed\n\n    Args:\n        dataset: A PyTorch [dataset][torch.utils.data.Dataset]\n        chunk_size: The size of the chunks for the resulting Dask arrays.\n        total_size: If the dataset does not implement len, provide the length\n            via this parameter. If None\n            the length of the dataset is inferred via accessing the dataset once.\n        resulting_dtype: The dtype of the resulting [dask.array.Array][dask.array.Array]\n\n    ??? Example\n        ```python\n        import torch\n        from torch.utils.data import TensorDataset\n        x = torch.rand((20, 3))\n        y = torch.rand((20, 1))\n        dataset = TensorDataset(x, y)\n        da_x, da_y = torch_dataset_to_dask_array(dataset, 4)\n        ```\n\n    Returns:\n        Tuple of Dask arrays corresponding to each tensor in the dataset.\n    \"\"\"\n\n    def _infer_data_len(d_set: Dataset):\n        try:\n            n_data = len(d_set)\n            if total_size is not None and n_data != total_size:\n                raise ValueError(\n                    f\"The number of samples in the dataset ({n_data}), derived \"\n                    f\"from calling \u00b4len\u00b4, does not match the provided \"\n                    f\"total number of samples ({total_size}). \"\n                    f\"Call the function without total_size.\"\n                )\n            return n_data\n        except TypeError as e:\n            err_msg = (\n                f\"Could not infer the number of samples in the dataset from \"\n                f\"calling \u00b4len\u00b4. Original error: {e}.\"\n            )\n            if total_size is not None:\n                logger.warning(\n                    err_msg\n                    + f\" Using the provided total number of samples {total_size}.\"\n                )\n                return total_size\n            else:\n                logger.warning(\n                    err_msg + f\" Infer the number of samples from the dataset, \"\n                    f\"via iterating the dataset once. \"\n                    f\"This might induce severe overhead, so consider\"\n                    f\"providing total_size, if you know the number of samples \"\n                    f\"beforehand.\"\n                )\n                idx = 0\n                while True:\n                    try:\n                        t = d_set[idx]\n                        if all(_t.numel() == 0 for _t in t):\n                            return idx\n                        idx += 1\n\n                    except IndexError:\n                        return idx\n\n    sample = dataset[0]\n    if not isinstance(sample, tuple):\n        sample = (sample,)\n\n    def _get_chunk(\n        start_idx: int, stop_idx: int, d_set: Dataset\n    ) -&gt; Tuple[torch.Tensor, ...]:\n        try:\n            t = d_set[start_idx:stop_idx]\n            if not isinstance(t, tuple):\n                t = (t,)\n            return t  # type:ignore\n        except Exception:\n            nested_tensor_list = [\n                [d_set[idx][k] for idx in range(start_idx, stop_idx)]\n                for k in range(len(sample))\n            ]\n            return tuple(map(torch.stack, nested_tensor_list))\n\n    n_samples = _infer_data_len(dataset)\n    chunk_indices = [\n        (i, min(i + chunk_size, n_samples)) for i in range(0, n_samples, chunk_size)\n    ]\n    delayed_dataset = dask.delayed(dataset)\n    delayed_chunks = [\n        dask.delayed(partial(_get_chunk, start, stop))(delayed_dataset)\n        for (start, stop) in chunk_indices\n    ]\n\n    delayed_arrays_dict: Dict[int, List[da.Array]] = {k: [] for k in range(len(sample))}\n\n    for chunk, (start, stop) in zip(delayed_chunks, chunk_indices):\n        for tensor_idx, sample_tensor in enumerate(sample):\n            delayed_tensor = da.from_delayed(\n                dask.delayed(lambda t: t.cpu().numpy())(chunk[tensor_idx]),\n                shape=(stop - start, *sample_tensor.shape),\n                dtype=resulting_dtype,\n            )\n\n            delayed_arrays_dict[tensor_idx].append(delayed_tensor)\n\n    return tuple(\n        da.concatenate(array_list) for array_list in delayed_arrays_dict.values()\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.empirical_cross_entropy_loss_fn","title":"empirical_cross_entropy_loss_fn","text":"<pre><code>empirical_cross_entropy_loss_fn(\n    model_output: Tensor, *args, **kwargs\n) -&gt; Tensor\n</code></pre> <p>Computes the empirical cross entropy loss of the model output. This is the cross entropy loss of the model output without the labels. The function takes all the usual arguments and keyword arguments of the cross entropy loss function, so that it is compatible with the PyTorch cross entropy loss function. However, it ignores everything except the first argument, which is the model output.</p> PARAMETER  DESCRIPTION <code>model_output</code> <p>The output of the model.</p> <p> TYPE: <code>Tensor</code> </p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def empirical_cross_entropy_loss_fn(\n    model_output: torch.Tensor, *args, **kwargs\n) -&gt; torch.Tensor:\n    \"\"\"\n    Computes the empirical cross entropy loss of the model output. This is the\n    cross entropy loss of the model output without the labels. The function takes\n    all the usual arguments and keyword arguments of the cross entropy loss\n    function, so that it is compatible with the PyTorch cross entropy loss\n    function. However, it ignores everything except the first argument, which is\n    the model output.\n\n    Args:\n        model_output: The output of the model.\n    \"\"\"\n    probs_ = torch.softmax(model_output, dim=1)\n    log_probs_ = torch.log(probs_)\n    log_probs_ = torch.where(\n        torch.isfinite(log_probs_), log_probs_, torch.zeros_like(log_probs_)\n    )\n    return torch.sum(log_probs_ * probs_.detach() ** 0.5)\n</code></pre>"},{"location":"api/pydvl/parallel/","title":"Parallel","text":""},{"location":"api/pydvl/parallel/#pydvl.parallel","title":"pydvl.parallel","text":"<p>This module provides a common interface to parallelization backends. The list of supported backends is here. Backends should be instantiated directly and passed to the respective valuation method.</p> <p>We use executors that implement the Executor interface to submit tasks in parallel. The basic high-level pattern is:</p> <pre><code>from pydvl.parallel import JoblibParallelBackend\n\nparallel_backend = JoblibParallelBackend()\nwith parallel_backend.executor(max_workers=2) as executor:\n    future = executor.submit(lambda x: x + 1, 1)\n    result = future.result()\nassert result == 2\n</code></pre> <p>Running a map-style job is also easy:</p> <pre><code>from pydvl.parallel import JoblibParallelBackend\n\nparallel_backend = JoblibParallelBackend()\nwith parallel_backend.executor(max_workers=2) as executor:\n    results = list(executor.map(lambda x: x + 1, range(5)))\nassert results == [1, 2, 3, 4, 5]\n</code></pre> <p>Passsing large objects</p> <p>When running tasks which accept heavy inputs, it is important to first use <code>put()</code> on the object and use the returned reference as argument to the callable within <code>submit()</code>. For example:   <pre><code>u_ref = parallel_backend.put(u)\n...\nexecutor.submit(task, utility=u)\n</code></pre> Note that <code>task()</code> does not need to be changed in any way: the backend will <code>get()</code> the object and pass it to the function upon invocation.</p> <p>There is an alternative map-reduce implementation MapReduceJob which internally uses joblib's higher level API with <code>Parallel()</code> which then indirectly also supports the use of Dask and Ray.</p>"},{"location":"api/pydvl/parallel/backend/","title":"Backend","text":""},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend","title":"pydvl.parallel.backend","text":""},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.CancellationPolicy","title":"CancellationPolicy","text":"<p>             Bases: <code>Flag</code></p> <p>Policy to use when cancelling futures after exiting an Executor.</p> <p>Note</p> <p>Not all backends support all policies.</p> ATTRIBUTE DESCRIPTION <code>NONE</code> <p>Do not cancel any futures.</p> <p> </p> <code>PENDING</code> <p>Cancel all pending futures, but not running ones.</p> <p> </p> <code>RUNNING</code> <p>Cancel all running futures, but not pending ones.</p> <p> </p> <code>ALL</code> <p>Cancel all pending and running futures.</p> <p> </p>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.ParallelBackend","title":"ParallelBackend","text":"<p>Abstract base class for all parallel backends.</p>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.ParallelBackend.executor","title":"executor  <code>abstractmethod</code> <code>classmethod</code>","text":"<pre><code>executor(\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.PENDING\n) -&gt; Executor\n</code></pre> <p>Returns a futures executor for the parallel backend.</p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>@classmethod\n@abstractmethod\ndef executor(\n    cls,\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.PENDING,\n) -&gt; Executor:\n    \"\"\"Returns a futures executor for the parallel backend.\"\"\"\n    ...\n</code></pre>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.init_parallel_backend","title":"init_parallel_backend","text":"<pre><code>init_parallel_backend(\n    config: ParallelConfig | None = None, backend_name: str | None = None\n) -&gt; ParallelBackend\n</code></pre> <p>Initializes the parallel backend and returns an instance of it.</p> <p>The following example creates a parallel backend instance with the default configuration, which is a local joblib backend.</p> <p>If you don't pass any arguments, then by default it will instantiate the JoblibParallelBackend:</p> Example <pre><code>parallel_backend = init_parallel_backend()\n</code></pre> <p>To create a parallel backend instance with for example <code>ray</code> as a backend, you can pass the backend name as a string:.</p> Example <pre><code>parallel_backend = init_parallel_backend(backend_name=\"ray\")\n</code></pre> <p>The following is an example of the deprecated way for instantiating a parallel backend:</p> Example <pre><code>config = ParallelConfig()\nparallel_backend = init_parallel_backend(config)\n</code></pre> PARAMETER  DESCRIPTION <code>backend_name</code> <p>Name of the backend to instantiate.</p> <p> TYPE: <code>str | None</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig | None</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef init_parallel_backend(\n    config: ParallelConfig | None = None, backend_name: str | None = None\n) -&gt; ParallelBackend:\n    \"\"\"Initializes the parallel backend and returns an instance of it.\n\n    The following example creates a parallel backend instance with the default\n    configuration, which is a local joblib backend.\n\n    If you don't pass any arguments, then by default it will instantiate\n    the JoblibParallelBackend:\n\n    ??? Example\n        ```python\n        parallel_backend = init_parallel_backend()\n        ```\n\n    To create a parallel backend instance with for example `ray` as a backend,\n    you can pass the backend name as a string:.\n\n    ??? Example\n        ```python\n        parallel_backend = init_parallel_backend(backend_name=\"ray\")\n        ```\n\n\n    The following is an example of the deprecated\n    way for instantiating a parallel backend:\n\n    ??? Example\n        ``` python\n        config = ParallelConfig()\n        parallel_backend = init_parallel_backend(config)\n        ```\n\n    Args:\n        backend_name: Name of the backend to instantiate.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n\n\n    \"\"\"\n    if backend_name is None:\n        if config is None:\n            backend_name = \"joblib\"\n        else:\n            backend_name = config.backend\n\n    try:\n        parallel_backend_cls = ParallelBackend.BACKENDS[backend_name]\n    except KeyError:\n        raise NotImplementedError(f\"Unexpected parallel backend {backend_name}\")\n    return parallel_backend_cls(config)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.available_cpus","title":"available_cpus","text":"<pre><code>available_cpus() -&gt; int\n</code></pre> <p>Platform-independent count of available cores.</p> <p>FIXME: do we really need this or is <code>os.cpu_count</code> enough? Is this portable?</p> RETURNS DESCRIPTION <code>int</code> <p>Number of cores, or 1 if it is not possible to determine.</p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>def available_cpus() -&gt; int:\n    \"\"\"Platform-independent count of available cores.\n\n    FIXME: do we really need this or is `os.cpu_count` enough? Is this portable?\n\n    Returns:\n        Number of cores, or 1 if it is not possible to determine.\n    \"\"\"\n    from platform import system\n\n    if system() != \"Linux\":\n        return os.cpu_count() or 1\n    return len(os.sched_getaffinity(0))  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/config/","title":"Config","text":""},{"location":"api/pydvl/parallel/config/#pydvl.parallel.config","title":"pydvl.parallel.config","text":""},{"location":"api/pydvl/parallel/config/#pydvl.parallel.config.ParallelConfig","title":"ParallelConfig  <code>dataclass</code>","text":"<pre><code>ParallelConfig(\n    backend: Literal[\"joblib\", \"ray\"] = \"joblib\",\n    address: Optional[Union[str, Tuple[str, int]]] = None,\n    n_cpus_local: Optional[int] = None,\n    logging_level: Optional[int] = None,\n    wait_timeout: float = 1.0,\n)\n</code></pre> <p>Configuration for parallel computation backend.</p> PARAMETER  DESCRIPTION <code>backend</code> <p>Type of backend to use. Defaults to 'joblib'</p> <p> TYPE: <code>Literal['joblib', 'ray']</code> DEFAULT: <code>'joblib'</code> </p> <code>address</code> <p>(DEPRECATED) Address of existing remote or local cluster to use.</p> <p> TYPE: <code>Optional[Union[str, Tuple[str, int]]]</code> DEFAULT: <code>None</code> </p> <code>n_cpus_local</code> <p>(DEPRECATED) Number of CPUs to use when creating a local ray cluster. This has no effect when using an existing ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>logging_level</code> <p>(DEPRECATED) Logging level for the parallel backend's worker.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>wait_timeout</code> <p>(DEPRECATED) Timeout in seconds for waiting on futures.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p>"},{"location":"api/pydvl/parallel/map_reduce/","title":"Map reduce","text":""},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce","title":"pydvl.parallel.map_reduce","text":"<p>This module contains a wrapper around joblib's <code>Parallel()</code> class that makes it easy to run map-reduce jobs.</p> <p>Deprecation</p> <p>This interface might be deprecated or changed in a future release before 1.0</p>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob","title":"MapReduceJob","text":"<pre><code>MapReduceJob(\n    inputs: Union[Collection[T], T],\n    map_func: MapFunction[R],\n    reduce_func: ReduceFunction[R] = identity,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    *,\n    map_kwargs: Optional[Dict] = None,\n    reduce_kwargs: Optional[Dict] = None,\n    n_jobs: int = -1,\n    timeout: Optional[float] = None\n)\n</code></pre> <p>             Bases: <code>Generic[T, R]</code></p> <p>Takes an embarrassingly parallel fun and runs it in <code>n_jobs</code> parallel jobs, splitting the data evenly into a number of chunks equal to the number of jobs.</p> <p>Typing information for objects of this class requires the type of the inputs that are split for <code>map_func</code> and the type of its output.</p> PARAMETER  DESCRIPTION <code>inputs</code> <p>The input that will be split and passed to <code>map_func</code>. if it's not a sequence object. It will be repeat <code>n_jobs</code> number of times.</p> <p> TYPE: <code>Union[Collection[T], T]</code> </p> <code>map_func</code> <p>Function that will be applied to the input chunks in each job.</p> <p> TYPE: <code>MapFunction[R]</code> </p> <code>reduce_func</code> <p>Function that will be applied to the results of <code>map_func</code> to reduce them.</p> <p> TYPE: <code>ReduceFunction[R]</code> DEFAULT: <code>identity</code> </p> <code>map_kwargs</code> <p>Keyword arguments that will be passed to <code>map_func</code> in each job. Alternatively, one can use functools.partial.</p> <p> TYPE: <code>Optional[Dict]</code> DEFAULT: <code>None</code> </p> <code>reduce_kwargs</code> <p>Keyword arguments that will be passed to <code>reduce_func</code> in each job. Alternatively, one can use functools.partial.</p> <p> TYPE: <code>Optional[Dict]</code> DEFAULT: <code>None</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to run. Does not accept 0</p> <p> TYPE: <code>int</code> DEFAULT: <code>-1</code> </p> Example <p>A simple usage example with 2 jobs:</p> <pre><code>&gt;&gt;&gt; from pydvl.parallel import MapReduceJob\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; map_reduce_job: MapReduceJob[np.ndarray, np.ndarray] = MapReduceJob(\n...     np.arange(5),\n...     map_func=np.sum,\n...     reduce_func=np.sum,\n...     n_jobs=2,\n... )\n&gt;&gt;&gt; map_reduce_job()\n10\n</code></pre> <p>When passed a single object as input, it will be repeated for each job: <pre><code>&gt;&gt;&gt; from pydvl.parallel import MapReduceJob\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; map_reduce_job: MapReduceJob[int, np.ndarray] = MapReduceJob(\n...     5,\n...     map_func=lambda x: np.array([x]),\n...     reduce_func=np.sum,\n...     n_jobs=2,\n... )\n&gt;&gt;&gt; map_reduce_job()\n10\n</code></pre></p> Source code in <code>src/pydvl/parallel/map_reduce.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef __init__(\n    self,\n    inputs: Union[Collection[T], T],\n    map_func: MapFunction[R],\n    reduce_func: ReduceFunction[R] = identity,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    *,\n    map_kwargs: Optional[Dict] = None,\n    reduce_kwargs: Optional[Dict] = None,\n    n_jobs: int = -1,\n    timeout: Optional[float] = None,\n):\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    self.parallel_backend = parallel_backend\n\n    self.timeout = timeout\n\n    self._n_jobs = -1\n    # This uses the setter defined below\n    self.n_jobs = n_jobs\n\n    self.inputs_ = inputs\n\n    self.map_kwargs = map_kwargs if map_kwargs is not None else dict()\n    self.reduce_kwargs = reduce_kwargs if reduce_kwargs is not None else dict()\n\n    self._map_func = reduce(maybe_add_argument, [\"job_id\", \"seed\"], map_func)\n    self._reduce_func = reduce_func\n</code></pre>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob.n_jobs","title":"n_jobs  <code>property</code> <code>writable</code>","text":"<pre><code>n_jobs: int\n</code></pre> <p>Effective number of jobs according to the used ParallelBackend instance.</p>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob.__call__","title":"__call__","text":"<pre><code>__call__(seed: Optional[Union[Seed, SeedSequence]] = None) -&gt; R\n</code></pre> <p>Runs the map-reduce job.</p> PARAMETER  DESCRIPTION <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Union[Seed, SeedSequence]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>R</code> <p>The result of the reduce function.</p> Source code in <code>src/pydvl/parallel/map_reduce.py</code> <pre><code>def __call__(\n    self,\n    seed: Optional[Union[Seed, SeedSequence]] = None,\n) -&gt; R:\n    \"\"\"\n    Runs the map-reduce job.\n\n    Args:\n        seed: Either an instance of a numpy random number generator or a seed for\n            it.\n\n    Returns:\n         The result of the reduce function.\n    \"\"\"\n    seed_seq = ensure_seed_sequence(seed)\n\n    if hasattr(self.parallel_backend, \"_joblib_backend_name\"):\n        backend = getattr(self.parallel_backend, \"_joblib_backend_name\")\n    else:\n        warnings.warn(\n            \"Parallel backend \"\n            f\"{self.parallel_backend.__class__.__name__}. \"\n            \"should have a `_joblib_backend_name` attribute in order to work \"\n            \"property with MapReduceJob. \"\n            \"Defaulting to joblib loky backend\"\n        )\n        backend = \"loky\"\n\n    with Parallel(backend=backend, prefer=\"processes\") as parallel:\n        chunks = self._chunkify(self.inputs_, n_chunks=self.n_jobs)\n        map_results: List[R] = parallel(\n            delayed(self._map_func)(\n                next_chunk, job_id=j, seed=seed, **self.map_kwargs\n            )\n            for j, (next_chunk, seed) in enumerate(\n                zip(chunks, seed_seq.spawn(len(chunks)))\n            )\n        )\n\n    reduce_results: R = self._reduce_func(map_results, **self.reduce_kwargs)\n    return reduce_results\n</code></pre>"},{"location":"api/pydvl/parallel/backends/","title":"Backends","text":""},{"location":"api/pydvl/parallel/backends/#pydvl.parallel.backends","title":"pydvl.parallel.backends","text":""},{"location":"api/pydvl/parallel/backends/joblib/","title":"Joblib","text":""},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib","title":"pydvl.parallel.backends.joblib","text":""},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib.JoblibParallelBackend","title":"JoblibParallelBackend","text":"<pre><code>JoblibParallelBackend(config: ParallelConfig | None = None)\n</code></pre> <p>             Bases: <code>ParallelBackend</code></p> <p>Class used to wrap joblib to make it transparent to algorithms.</p> <p>Example</p> <pre><code>from pydvl.parallel import JoblibParallelBackend\nparallel_backend = JoblibParallelBackend()\n</code></pre> Source code in <code>src/pydvl/parallel/backends/joblib.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": None},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef __init__(self, config: ParallelConfig | None = None) -&gt; None:\n    n_jobs: int | None = None\n    if config is not None:\n        n_jobs = config.n_cpus_local\n    self.config = {\n        \"n_jobs\": n_jobs,\n    }\n</code></pre>"},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib.JoblibParallelBackend.executor","title":"executor  <code>classmethod</code>","text":"<pre><code>executor(\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.NONE\n) -&gt; Executor\n</code></pre> <p>Returns a futures executor for the parallel backend.</p> <p>Example</p> <pre><code>from pydvl.parallel import JoblibParallelBackend\nparallel_backend = JoblibParallelBackend()\nwith parallel_backend.executor() as executor:\n    executor.submit(...)\n</code></pre> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of parallel workers.</p> <p> TYPE: <code>int | None</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig | None</code> DEFAULT: <code>None</code> </p> <code>cancel_futures</code> <p>Policy to use when cancelling futures after exiting an Executor.</p> <p> TYPE: <code>CancellationPolicy | bool</code> DEFAULT: <code>NONE</code> </p> RETURNS DESCRIPTION <code>Executor</code> <p>Instance of _ReusablePoolExecutor.</p> Source code in <code>src/pydvl/parallel/backends/joblib.py</code> <pre><code>@classmethod\ndef executor(\n    cls,\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.NONE,\n) -&gt; Executor:\n    \"\"\"Returns a futures executor for the parallel backend.\n\n    !!! Example\n        ``` python\n        from pydvl.parallel import JoblibParallelBackend\n        parallel_backend = JoblibParallelBackend()\n        with parallel_backend.executor() as executor:\n            executor.submit(...)\n        ```\n\n    Args:\n        max_workers: Maximum number of parallel workers.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        cancel_futures: Policy to use when cancelling futures\n            after exiting an Executor.\n\n    Returns:\n        Instance of [_ReusablePoolExecutor][joblib.externals.loky.reusable_executor._ReusablePoolExecutor].\n    \"\"\"\n    if config is not None:\n        warnings.warn(\n            \"The `JoblibParallelBackend` uses deprecated arguments: \"\n            \"`config`. They were deprecated since v0.9.0 \"\n            \"and will be removed in v0.10.0.\",\n            FutureWarning,\n        )\n\n    if cancel_futures not in (CancellationPolicy.NONE, False):\n        warnings.warn(\n            \"Cancellation of futures is not supported by the joblib backend\",\n        )\n    return cast(Executor, get_reusable_executor(max_workers=max_workers))\n</code></pre>"},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib.JoblibParallelBackend.wrap","title":"wrap","text":"<pre><code>wrap(fun: Callable, **kwargs) -&gt; Callable\n</code></pre> <p>Wraps a function as a joblib delayed.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>the function to wrap</p> <p> TYPE: <code>Callable</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>The delayed function.</p> Source code in <code>src/pydvl/parallel/backends/joblib.py</code> <pre><code>def wrap(self, fun: Callable, **kwargs) -&gt; Callable:\n    \"\"\"Wraps a function as a joblib delayed.\n\n    Args:\n        fun: the function to wrap\n\n    Returns:\n        The delayed function.\n    \"\"\"\n    return delayed(fun)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backends/ray/","title":"Ray","text":""},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray","title":"pydvl.parallel.backends.ray","text":""},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray.RayParallelBackend","title":"RayParallelBackend","text":"<pre><code>RayParallelBackend(config: ParallelConfig | None = None)\n</code></pre> <p>             Bases: <code>ParallelBackend</code></p> <p>Class used to wrap ray to make it transparent to algorithms.</p> <p>Example</p> <pre><code>import ray\nfrom pydvl.parallel import RayParallelBackend\nray.init()\nparallel_backend = RayParallelBackend()\n</code></pre> Source code in <code>src/pydvl/parallel/backends/ray.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": None},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef __init__(self, config: ParallelConfig | None = None) -&gt; None:\n    if not ray.is_initialized():\n        raise RuntimeError(\n            \"Starting from v0.9.0, ray is no longer automatically initialized. \"\n            \"Please use `ray.init()` with the desired configuration \"\n            \"before using this class.\"\n        )\n    # Register ray joblib backend\n    register_ray()\n</code></pre>"},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray.RayParallelBackend.executor","title":"executor  <code>classmethod</code>","text":"<pre><code>executor(\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.PENDING\n) -&gt; Executor\n</code></pre> <p>Returns a futures executor for the parallel backend.</p> <p>Example</p> <pre><code>import ray\nfrom pydvl.parallel import RayParallelBackend\nray.init()\nparallel_backend = RayParallelBackend()\nwith parallel_backend.executor() as executor:\n    executor.submit(...)\n</code></pre> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of parallel workers.</p> <p> TYPE: <code>int | None</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig | None</code> DEFAULT: <code>None</code> </p> <code>cancel_futures</code> <p>Policy to use when cancelling futures after exiting an Executor.</p> <p> TYPE: <code>CancellationPolicy | bool</code> DEFAULT: <code>PENDING</code> </p> RETURNS DESCRIPTION <code>Executor</code> <p>Instance of RayExecutor.</p> Source code in <code>src/pydvl/parallel/backends/ray.py</code> <pre><code>@classmethod\ndef executor(\n    cls,\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.PENDING,\n) -&gt; Executor:\n    \"\"\"Returns a futures executor for the parallel backend.\n\n    !!! Example\n        ``` python\n        import ray\n        from pydvl.parallel import RayParallelBackend\n        ray.init()\n        parallel_backend = RayParallelBackend()\n        with parallel_backend.executor() as executor:\n            executor.submit(...)\n        ```\n\n    Args:\n        max_workers: Maximum number of parallel workers.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        cancel_futures: Policy to use when cancelling futures\n            after exiting an Executor.\n\n    Returns:\n        Instance of [RayExecutor][pydvl.parallel.futures.ray.RayExecutor].\n    \"\"\"\n    # Imported here to avoid circular import errors\n    from pydvl.parallel.futures.ray import RayExecutor\n\n    if config is not None:\n        warnings.warn(\n            \"The `RayParallelBackend` uses deprecated arguments: \"\n            \"`config`. They were deprecated since v0.9.0 \"\n            \"and will be removed in v0.10.0.\",\n            FutureWarning,\n        )\n\n    return RayExecutor(max_workers, cancel_futures=cancel_futures)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray.RayParallelBackend.wrap","title":"wrap","text":"<pre><code>wrap(fun: Callable, **kwargs) -&gt; Callable\n</code></pre> <p>Wraps a function as a ray remote.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>the function to wrap</p> <p> TYPE: <code>Callable</code> </p> <code>kwargs</code> <p>keyword arguments to pass to @ray.remote</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>The <code>.remote</code> method of the ray <code>RemoteFunction</code>.</p> Source code in <code>src/pydvl/parallel/backends/ray.py</code> <pre><code>def wrap(self, fun: Callable, **kwargs) -&gt; Callable:\n    \"\"\"Wraps a function as a ray remote.\n\n    Args:\n        fun: the function to wrap\n        kwargs: keyword arguments to pass to @ray.remote\n\n    Returns:\n        The `.remote` method of the ray `RemoteFunction`.\n    \"\"\"\n    if len(kwargs) &gt; 0:\n        return ray.remote(**kwargs)(fun).remote  # type: ignore\n    return ray.remote(fun).remote  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/futures/","title":"Futures","text":""},{"location":"api/pydvl/parallel/futures/#pydvl.parallel.futures","title":"pydvl.parallel.futures","text":""},{"location":"api/pydvl/parallel/futures/#pydvl.parallel.futures.init_executor","title":"init_executor","text":"<pre><code>init_executor(\n    max_workers: Optional[int] = None,\n    config: ParallelConfig = ParallelConfig(),\n    **kwargs\n) -&gt; Generator[Executor, None, None]\n</code></pre> <p>Initializes a futures executor for the given parallel configuration.</p> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of concurrent tasks.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>kwargs</code> <p>Other optional parameter that will be passed to the executor.</p> <p> DEFAULT: <code>{}</code> </p> Examples <p><pre><code>from pydvl.parallel.futures import init_executor, ParallelConfig\n\nconfig = ParallelConfig(backend=\"ray\")\nwith init_executor(max_workers=1, config=config) as executor:\n    future = executor.submit(lambda x: x + 1, 1)\n    result = future.result()\nassert result == 2\n</code></pre> <pre><code>from pydvl.parallel.futures import init_executor\nwith init_executor() as executor:\n    results = list(executor.map(lambda x: x + 1, range(5)))\nassert results == [1, 2, 3, 4, 5]\n</code></pre></p> Source code in <code>src/pydvl/parallel/futures/__init__.py</code> <pre><code>@contextmanager\n@deprecated(\n    target=None,\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef init_executor(\n    max_workers: Optional[int] = None,\n    config: ParallelConfig = ParallelConfig(),\n    **kwargs,\n) -&gt; Generator[Executor, None, None]:\n    \"\"\"Initializes a futures executor for the given parallel configuration.\n\n    Args:\n        max_workers: Maximum number of concurrent tasks.\n        config: instance of [ParallelConfig][pydvl.utils.config.ParallelConfig]\n            with cluster address, number of cpus, etc.\n        kwargs: Other optional parameter that will be passed to the executor.\n\n\n    ??? Examples\n        ``` python\n        from pydvl.parallel.futures import init_executor, ParallelConfig\n\n        config = ParallelConfig(backend=\"ray\")\n        with init_executor(max_workers=1, config=config) as executor:\n            future = executor.submit(lambda x: x + 1, 1)\n            result = future.result()\n        assert result == 2\n        ```\n        ``` python\n        from pydvl.parallel.futures import init_executor\n        with init_executor() as executor:\n            results = list(executor.map(lambda x: x + 1, range(5)))\n        assert results == [1, 2, 3, 4, 5]\n        ```\n    \"\"\"\n    try:\n        cls = ParallelBackend.BACKENDS[config.backend]\n        with cls.executor(max_workers=max_workers, config=config, **kwargs) as e:\n            yield e\n    except KeyError:\n        raise NotImplementedError(f\"Unexpected parallel backend {config.backend}\")\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/","title":"Ray","text":""},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray","title":"pydvl.parallel.futures.ray","text":""},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor","title":"RayExecutor","text":"<pre><code>RayExecutor(\n    max_workers: Optional[int] = None,\n    *,\n    config: Optional[ParallelConfig] = None,\n    cancel_futures: Union[CancellationPolicy, bool] = CancellationPolicy.ALL\n)\n</code></pre> <p>             Bases: <code>Executor</code></p> <p>Asynchronous executor using Ray that implements the concurrent.futures API.</p> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of concurrent tasks. Each task can request itself any number of vCPUs. You must ensure the product of this value and the n_cpus_per_job parameter passed to submit() does not exceed available cluster resources. If set to <code>None</code>, it will default to the total number of vCPUs in the ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>cancel_futures</code> <p>Select which futures will be cancelled when exiting this context manager. <code>Pending</code> is the default, which will cancel all pending futures, but not running ones, as done by concurrent.futures.ProcessPoolExecutor. Additionally, <code>All</code> cancels all pending and running futures, and <code>None</code> doesn't cancel any. See CancellationPolicy</p> <p> TYPE: <code>Union[CancellationPolicy, bool]</code> DEFAULT: <code>ALL</code> </p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": None},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef __init__(\n    self,\n    max_workers: Optional[int] = None,\n    *,\n    config: Optional[ParallelConfig] = None,\n    cancel_futures: Union[CancellationPolicy, bool] = CancellationPolicy.ALL,\n):\n    if max_workers is not None:\n        if max_workers &lt;= 0:\n            raise ValueError(\"max_workers must be greater than 0\")\n        max_workers = max_workers\n\n    if isinstance(cancel_futures, CancellationPolicy):\n        self._cancel_futures = cancel_futures\n    else:\n        self._cancel_futures = (\n            CancellationPolicy.PENDING\n            if cancel_futures\n            else CancellationPolicy.NONE\n        )\n\n    if not ray.is_initialized():\n        raise RuntimeError(\n            \"Starting from v0.9.0, ray is no longer automatically initialized. \"\n            \"Please use `ray.init()` with the desired configuration \"\n            \"before using this class.\"\n        )\n\n    self._max_workers = max_workers\n    if self._max_workers is None:\n        self._max_workers = int(ray._private.state.cluster_resources()[\"CPU\"])\n\n    self._shutdown = False\n    self._shutdown_lock = threading.Lock()\n    self._queue_lock = threading.Lock()\n    self._work_queue: \"queue.Queue[Optional[_WorkItem]]\" = queue.Queue(\n        maxsize=self._max_workers\n    )\n    self._pending_queue: \"queue.SimpleQueue[Optional[_WorkItem]]\" = (\n        queue.SimpleQueue()\n    )\n\n    # Work Item Manager Thread\n    self._work_item_manager_thread: Optional[_WorkItemManagerThread] = None\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.submit","title":"submit","text":"<pre><code>submit(fn: Callable[..., T], *args, **kwargs) -&gt; Future[T]\n</code></pre> <p>Submits a callable to be executed with the given arguments.</p> <p>Schedules the callable to be executed as fn(*args, **kwargs) and returns a Future instance representing the execution of the callable.</p> PARAMETER  DESCRIPTION <code>fn</code> <p>Callable.</p> <p> TYPE: <code>Callable[..., T]</code> </p> <code>args</code> <p>Positional arguments that will be passed to <code>fn</code>.</p> <p> DEFAULT: <code>()</code> </p> <code>kwargs</code> <p>Keyword arguments that will be passed to <code>fn</code>. It can also optionally contain options for the ray remote function as a dictionary as the keyword argument <code>remote_function_options</code>.</p> <p> DEFAULT: <code>{}</code> </p> <p>Returns:     A Future representing the given call.</p> RAISES DESCRIPTION <code>RuntimeError</code> <p>If a task is submitted after the executor has been shut down.</p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def submit(self, fn: Callable[..., T], *args, **kwargs) -&gt; \"Future[T]\":\n    r\"\"\"Submits a callable to be executed with the given arguments.\n\n    Schedules the callable to be executed as fn(\\*args, \\**kwargs)\n    and returns a Future instance representing the execution of the callable.\n\n    Args:\n        fn: Callable.\n        args: Positional arguments that will be passed to `fn`.\n        kwargs: Keyword arguments that will be passed to `fn`.\n            It can also optionally contain options for the ray remote function\n            as a dictionary as the keyword argument `remote_function_options`.\n    Returns:\n        A Future representing the given call.\n\n    Raises:\n        RuntimeError: If a task is submitted after the executor has been shut down.\n    \"\"\"\n    with self._shutdown_lock:\n        logger.debug(\"executor acquired shutdown lock\")\n        if self._shutdown:\n            raise RuntimeError(\"cannot schedule new futures after shutdown\")\n\n        logging.debug(\"Creating future and putting work item in work queue\")\n        future: \"Future[T]\" = Future()\n        remote_function_options = kwargs.pop(\"remote_function_options\", None)\n        w = _WorkItem(\n            future,\n            fn,\n            args,\n            kwargs,\n            remote_function_options=remote_function_options,\n        )\n        self._put_work_item_in_queue(w)\n        # We delay starting the thread until the first call to submit\n        self._start_work_item_manager_thread()\n        return future\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.shutdown","title":"shutdown","text":"<pre><code>shutdown(wait: bool = True, *, cancel_futures: Optional[bool] = None) -&gt; None\n</code></pre> <p>Clean up the resources associated with the Executor.</p> <p>This method tries to mimic the behaviour of Executor.shutdown while allowing one more value for <code>cancel_futures</code> which instructs it to use the CancellationPolicy defined upon construction.</p> PARAMETER  DESCRIPTION <code>wait</code> <p>Whether to wait for pending futures to finish.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>cancel_futures</code> <p>Overrides the executor's default policy for cancelling futures on exit. If <code>True</code>, all pending futures are cancelled, and if <code>False</code>, no futures are cancelled. If <code>None</code> (default), the executor's policy set at initialization is used.</p> <p> TYPE: <code>Optional[bool]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def shutdown(\n    self, wait: bool = True, *, cancel_futures: Optional[bool] = None\n) -&gt; None:\n    \"\"\"Clean up the resources associated with the Executor.\n\n    This method tries to mimic the behaviour of\n    [Executor.shutdown][concurrent.futures.Executor.shutdown]\n    while allowing one more value for ``cancel_futures`` which instructs it\n    to use the [CancellationPolicy][pydvl.parallel.backend.CancellationPolicy]\n    defined upon construction.\n\n    Args:\n        wait: Whether to wait for pending futures to finish.\n        cancel_futures: Overrides the executor's default policy for\n            cancelling futures on exit. If ``True``, all pending futures are\n            cancelled, and if ``False``, no futures are cancelled. If ``None``\n            (default), the executor's policy set at initialization is used.\n    \"\"\"\n    logger.debug(\"executor shutting down\")\n    with self._shutdown_lock:\n        logger.debug(\"executor acquired shutdown lock\")\n        self._shutdown = True\n        self._cancel_futures = {\n            None: self._cancel_futures,\n            True: CancellationPolicy.PENDING,\n            False: CancellationPolicy.NONE,\n        }[cancel_futures]\n\n    if wait:\n        logger.debug(\"executor waiting for futures to finish\")\n        if self._work_item_manager_thread is not None:\n            # Putting None in the queue to signal\n            # to work item manager thread that we are shutting down\n            self._put_work_item_in_queue(None)\n            logger.debug(\n                \"executor waiting for work item manager thread to terminate\"\n            )\n            self._work_item_manager_thread.join()\n        # To reduce the risk of opening too many files, remove references to\n        # objects that use file descriptors.\n        self._work_item_manager_thread = None\n        del self._work_queue\n        del self._pending_queue\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.__exit__","title":"__exit__","text":"<pre><code>__exit__(exc_type, exc_val, exc_tb)\n</code></pre> <p>Exit the runtime context related to the RayExecutor object.</p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def __exit__(self, exc_type, exc_val, exc_tb):\n    \"\"\"Exit the runtime context related to the RayExecutor object.\"\"\"\n    self.shutdown()\n    return False\n</code></pre>"},{"location":"api/pydvl/reporting/","title":"Reporting","text":""},{"location":"api/pydvl/reporting/#pydvl.reporting","title":"pydvl.reporting","text":""},{"location":"api/pydvl/reporting/plots/","title":"Plots","text":""},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots","title":"pydvl.reporting.plots","text":""},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.shaded_mean_std","title":"shaded_mean_std","text":"<pre><code>shaded_mean_std(\n    data: ndarray,\n    abscissa: Optional[Sequence[Any]] = None,\n    num_std: float = 1.0,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    title: Optional[str] = None,\n    xlabel: Optional[str] = None,\n    ylabel: Optional[str] = None,\n    ax: Optional[Axes] = None,\n    **kwargs\n) -&gt; Axes\n</code></pre> <p>The usual mean \\(\\pm\\) std deviation plot to aggregate runs of experiments.</p> <p>Deprecation notice</p> <p>This function is bogus and will be removed in the future in favour of properly computed confidence intervals.</p> PARAMETER  DESCRIPTION <code>data</code> <p>axis 0 is to be aggregated on (e.g. runs) and axis 1 is the data for each run.</p> <p> TYPE: <code>ndarray</code> </p> <code>abscissa</code> <p>values for the x-axis. Leave empty to use increasing integers.</p> <p> TYPE: <code>Optional[Sequence[Any]]</code> DEFAULT: <code>None</code> </p> <code>num_std</code> <p>number of standard deviations to shade around the mean.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>mean_color</code> <p>color for the mean</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'dodgerblue'</code> </p> <code>shade_color</code> <p>color for the shaded region</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'lightblue'</code> </p> <code>title</code> <p>Title text. To use mathematics, use LaTeX notation.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>xlabel</code> <p>Text for the horizontal axis.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ylabel</code> <p>Text for the vertical axis</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ax</code> <p>If passed, axes object into which to insert the figure. Otherwise, a new figure is created and returned</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>these are forwarded to the ax.plot() call for the mean.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Axes</code> <p>The axes used (or created)</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>@deprecated(target=None, deprecated_in=\"0.7.1\", remove_in=\"0.9.0\")\ndef shaded_mean_std(\n    data: np.ndarray,\n    abscissa: Optional[Sequence[Any]] = None,\n    num_std: float = 1.0,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    title: Optional[str] = None,\n    xlabel: Optional[str] = None,\n    ylabel: Optional[str] = None,\n    ax: Optional[Axes] = None,\n    **kwargs,\n) -&gt; Axes:\n    r\"\"\"The usual mean \\(\\pm\\) std deviation plot to aggregate runs of\n    experiments.\n\n    !!! warning \"Deprecation notice\"\n        This function is bogus and will be removed in the future in favour of\n        properly computed confidence intervals.\n\n    Args:\n        data: axis 0 is to be aggregated on (e.g. runs) and axis 1 is the\n            data for each run.\n        abscissa: values for the x-axis. Leave empty to use increasing integers.\n        num_std: number of standard deviations to shade around the mean.\n        mean_color: color for the mean\n        shade_color: color for the shaded region\n        title: Title text. To use mathematics, use LaTeX notation.\n        xlabel: Text for the horizontal axis.\n        ylabel: Text for the vertical axis\n        ax: If passed, axes object into which to insert the figure. Otherwise,\n            a new figure is created and returned\n        kwargs: these are forwarded to the ax.plot() call for the mean.\n\n    Returns:\n        The axes used (or created)\n    \"\"\"\n    assert len(data.shape) == 2\n    mean = data.mean(axis=0)\n    std = num_std * data.std(axis=0)\n\n    if ax is None:\n        fig, ax = plt.subplots()\n    if abscissa is None:\n        abscissa = list(range(data.shape[1]))\n\n    ax.fill_between(abscissa, mean - std, mean + std, alpha=0.3, color=shade_color)\n    ax.plot(abscissa, mean, color=mean_color, **kwargs)\n\n    ax.set_title(title)\n    ax.set_xlabel(xlabel)\n    ax.set_ylabel(ylabel)\n\n    return ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_ci_array","title":"plot_ci_array","text":"<pre><code>plot_ci_array(\n    data: NDArray,\n    level: float,\n    type: Literal[\"normal\", \"t\", \"auto\"] = \"normal\",\n    abscissa: Optional[Sequence[str]] = None,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    ax: Optional[Axes] = None,\n    **kwargs\n) -&gt; Axes\n</code></pre> <p>Plot values and a confidence interval from a 2D array.</p> <p>Supported intervals are based on the normal and the t distributions.</p> PARAMETER  DESCRIPTION <code>data</code> <p>A 2D array with M different values for each of the N indices.</p> <p> TYPE: <code>NDArray</code> </p> <code>level</code> <p>The confidence level.</p> <p> TYPE: <code>float</code> </p> <code>type</code> <p>The type of confidence interval to use.</p> <p> TYPE: <code>Literal['normal', 't', 'auto']</code> DEFAULT: <code>'normal'</code> </p> <code>abscissa</code> <p>The values for the x-axis. Leave empty to use increasing integers.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>mean_color</code> <p>The color of the mean line.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'dodgerblue'</code> </p> <code>shade_color</code> <p>The color of the confidence interval.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'lightblue'</code> </p> <code>ax</code> <p>If passed, axes object into which to insert the figure. Otherwise, a new figure is created and the axes returned.</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>**kwargs</code> <p>Additional arguments to pass to the plot function.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Axes</code> <p>The matplotlib axes.</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_ci_array(\n    data: NDArray,\n    level: float,\n    type: Literal[\"normal\", \"t\", \"auto\"] = \"normal\",\n    abscissa: Optional[Sequence[str]] = None,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    ax: Optional[plt.Axes] = None,\n    **kwargs,\n) -&gt; plt.Axes:\n    \"\"\"Plot values and a confidence interval from a 2D array.\n\n    Supported intervals are based on the normal and the t distributions.\n\n    Args:\n        data: A 2D array with M different values for each of the N indices.\n        level: The confidence level.\n        type: The type of confidence interval to use.\n        abscissa: The values for the x-axis. Leave empty to use increasing\n            integers.\n        mean_color: The color of the mean line.\n        shade_color: The color of the confidence interval.\n        ax: If passed, axes object into which to insert the figure. Otherwise,\n            a new figure is created and the axes returned.\n        **kwargs: Additional arguments to pass to the plot function.\n\n    Returns:\n        The matplotlib axes.\n    \"\"\"\n\n    m, n = data.shape\n\n    means = np.mean(data, axis=0)\n    variances = np.var(data, axis=0, ddof=1)\n\n    dummy = ValuationResult[np.int_, np.object_](\n        algorithm=\"dummy\",\n        values=means,\n        variances=variances,\n        counts=np.ones_like(means, dtype=np.int_) * m,\n        indices=np.arange(n),\n        data_names=np.array(abscissa, dtype=str)\n        if abscissa is not None\n        else np.arange(n, dtype=str),\n    )\n\n    return plot_ci_values(\n        dummy,\n        level=level,\n        type=type,\n        mean_color=mean_color,\n        shade_color=shade_color,\n        ax=ax,\n        **kwargs,\n    )\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_ci_values","title":"plot_ci_values","text":"<pre><code>plot_ci_values(\n    values: ValuationResult,\n    level: float,\n    type: Literal[\"normal\", \"t\", \"auto\"] = \"auto\",\n    abscissa: Optional[Sequence[str]] = None,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    ax: Optional[Axes] = None,\n    **kwargs\n)\n</code></pre> <p>Plot values and a confidence interval.</p> <p>Uses <code>values.data_names</code> for the x-axis.</p> <p>Supported intervals are based on the normal and the t distributions.</p> PARAMETER  DESCRIPTION <code>values</code> <p>The valuation result.</p> <p> TYPE: <code>ValuationResult</code> </p> <code>level</code> <p>The confidence level.</p> <p> TYPE: <code>float</code> </p> <code>type</code> <p>The type of confidence interval to use. If \"auto\", uses \"norm\" if the minimum number of updates for all indices is greater than 30, otherwise uses \"t\".</p> <p> TYPE: <code>Literal['normal', 't', 'auto']</code> DEFAULT: <code>'auto'</code> </p> <code>abscissa</code> <p>The values for the x-axis. Leave empty to use increasing integers.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>mean_color</code> <p>The color of the mean line.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'dodgerblue'</code> </p> <code>shade_color</code> <p>The color of the confidence interval.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'lightblue'</code> </p> <code>ax</code> <p>If passed, axes object into which to insert the figure. Otherwise, a new figure is created and the axes returned.</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>**kwargs</code> <p>Additional arguments to pass to the plot function.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <p>The matplotlib axes.</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_ci_values(\n    values: ValuationResult,\n    level: float,\n    type: Literal[\"normal\", \"t\", \"auto\"] = \"auto\",\n    abscissa: Optional[Sequence[str]] = None,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    ax: Optional[plt.Axes] = None,\n    **kwargs,\n):\n    \"\"\"Plot values and a confidence interval.\n\n    Uses `values.data_names` for the x-axis.\n\n    Supported intervals are based on the normal and the t distributions.\n\n    Args:\n        values: The valuation result.\n        level: The confidence level.\n        type: The type of confidence interval to use. If \"auto\", uses \"norm\" if\n            the minimum number of updates for all indices is greater than 30,\n            otherwise uses \"t\".\n        abscissa: The values for the x-axis. Leave empty to use increasing\n            integers.\n        mean_color: The color of the mean line.\n        shade_color: The color of the confidence interval.\n        ax: If passed, axes object into which to insert the figure. Otherwise,\n            a new figure is created and the axes returned.\n        **kwargs: Additional arguments to pass to the plot function.\n\n    Returns:\n        The matplotlib axes.\n    \"\"\"\n\n    ppfs = {\n        \"normal\": norm.ppf,\n        \"t\": partial(t.ppf, df=values.counts - 1),\n        \"auto\": norm.ppf\n        if np.min(values.counts) &gt; 30\n        else partial(t.ppf, df=values.counts - 1),\n    }\n\n    try:\n        score = ppfs[type](1 - level / 2)\n    except KeyError:\n        raise ValueError(\n            f\"Unknown confidence interval type requested: {type}.\"\n        ) from None\n\n    if abscissa is None:\n        abscissa = [str(i) for i, _ in enumerate(values)]\n    bound = score * values.stderr\n\n    if ax is None:\n        fig, ax = plt.subplots()\n\n    ax.fill_between(\n        abscissa,\n        values.values - bound,\n        values.values + bound,\n        alpha=0.3,\n        color=shade_color,\n    )\n    ax.plot(abscissa, values.values, color=mean_color, **kwargs)\n    return ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.spearman_correlation","title":"spearman_correlation","text":"<pre><code>spearman_correlation(vv: List[OrderedDict], num_values: int, pvalue: float)\n</code></pre> <p>Simple matrix plots with spearman correlation for each pair in vv.</p> PARAMETER  DESCRIPTION <code>vv</code> <p>list of OrderedDicts with index: value. Spearman correlation is computed for the keys.</p> <p> TYPE: <code>List[OrderedDict]</code> </p> <code>num_values</code> <p>Use only these many values from the data (from the start of the OrderedDicts)</p> <p> TYPE: <code>int</code> </p> <code>pvalue</code> <p>correlation coefficients for which the p-value is below the threshold <code>pvalue/len(vv)</code> will be discarded.</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def spearman_correlation(vv: List[OrderedDict], num_values: int, pvalue: float):\n    \"\"\"Simple matrix plots with spearman correlation for each pair in vv.\n\n    Args:\n        vv: list of OrderedDicts with index: value. Spearman correlation\n            is computed for the keys.\n        num_values: Use only these many values from the data (from the start\n            of the OrderedDicts)\n        pvalue: correlation coefficients for which the p-value is below the\n            threshold `pvalue/len(vv)` will be discarded.\n    \"\"\"\n    r: np.ndarray = np.ndarray((len(vv), len(vv)))\n    p: np.ndarray = np.ndarray((len(vv), len(vv)))\n    for i, a in enumerate(vv):\n        for j, b in enumerate(vv):\n            from scipy.stats._stats_py import SpearmanrResult\n\n            spearman: SpearmanrResult = sp.stats.spearmanr(\n                list(a.keys())[:num_values], list(b.keys())[:num_values]\n            )\n            r[i][j] = (\n                spearman.correlation if spearman.pvalue &lt; pvalue / len(vv) else np.nan\n            )  # Bonferroni correction\n            p[i][j] = spearman.pvalue\n    fig, axs = plt.subplots(1, 2, figsize=(16, 7))\n    plot1 = axs[0].matshow(r, vmin=-1, vmax=1)\n    axs[0].set_title(f\"Spearman correlation (top {num_values} values)\")\n    axs[0].set_xlabel(\"Runs\")\n    axs[0].set_ylabel(\"Runs\")\n    fig.colorbar(plot1, ax=axs[0])\n    plot2 = axs[1].matshow(p, vmin=0, vmax=1)\n    axs[1].set_title(\"p-value\")\n    axs[1].set_xlabel(\"Runs\")\n    axs[1].set_ylabel(\"Runs\")\n    fig.colorbar(plot2, ax=axs[1])\n\n    return fig\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_shapley","title":"plot_shapley","text":"<pre><code>plot_shapley(\n    df: DataFrame,\n    *,\n    level: float = 0.05,\n    ax: Optional[Axes] = None,\n    title: Optional[str] = None,\n    xlabel: Optional[str] = None,\n    ylabel: Optional[str] = None,\n    prefix: Optional[str] = \"data_value\"\n) -&gt; Axes\n</code></pre> <p>Plots the shapley values, as returned from compute_shapley_values, with error bars corresponding to an \\(\\alpha\\)-level Normal confidence interval.</p> PARAMETER  DESCRIPTION <code>df</code> <p>dataframe with the shapley values</p> <p> TYPE: <code>DataFrame</code> </p> <code>level</code> <p>confidence level for the error bars</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.05</code> </p> <code>ax</code> <p>axes to plot on or None if a new subplots should be created</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>title</code> <p>string, title of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>xlabel</code> <p>string, x label of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ylabel</code> <p>string, y label of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Axes</code> <p>The axes created or used</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_shapley(\n    df: pd.DataFrame,\n    *,\n    level: float = 0.05,\n    ax: Optional[plt.Axes] = None,\n    title: Optional[str] = None,\n    xlabel: Optional[str] = None,\n    ylabel: Optional[str] = None,\n    prefix: Optional[str] = \"data_value\",\n) -&gt; plt.Axes:\n    r\"\"\"Plots the shapley values, as returned from\n    [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values],\n    with error bars corresponding to an $\\alpha$-level Normal confidence\n    interval.\n\n    Args:\n        df: dataframe with the shapley values\n        level: confidence level for the error bars\n        ax: axes to plot on or None if a new subplots should be created\n        title: string, title of the plot\n        xlabel: string, x label of the plot\n        ylabel: string, y label of the plot\n\n    Returns:\n        The axes created or used\n    \"\"\"\n    if ax is None:\n        _, ax = plt.subplots()\n\n    yerr = norm.ppf(1 - level / 2) * df[f\"{prefix}_stderr\"]\n\n    ax.errorbar(x=df.index, y=df[prefix], yerr=yerr, fmt=\"o\", capsize=6)\n    ax.set_xlabel(xlabel)\n    ax.set_ylabel(ylabel)\n    ax.set_title(title)\n    plt.xticks(rotation=60)\n    return ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_influence_distribution","title":"plot_influence_distribution","text":"<pre><code>plot_influence_distribution(\n    influences: NDArray[float_], index: int, title_extra: str = \"\"\n) -&gt; Axes\n</code></pre> <p>Plots the histogram of the influence that all samples in the training set have over a single sample index.</p> PARAMETER  DESCRIPTION <code>influences</code> <p>array of influences (training samples x test samples)</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>index</code> <p>Index of the test sample for which the influences   will be plotted.</p> <p> TYPE: <code>int</code> </p> <code>title_extra</code> <p>Additional text that will be appended to the title.</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_influence_distribution(\n    influences: NDArray[np.float_], index: int, title_extra: str = \"\"\n) -&gt; plt.Axes:\n    \"\"\"Plots the histogram of the influence that all samples in the training set\n    have over a single sample index.\n\n    Args:\n       influences: array of influences (training samples x test samples)\n       index: Index of the test sample for which the influences\n            will be plotted.\n       title_extra: Additional text that will be appended to the title.\n    \"\"\"\n    _, ax = plt.subplots()\n    ax.hist(influences[:, index], alpha=0.7)\n    ax.set_xlabel(\"Influence values\")\n    ax.set_ylabel(\"Number of samples\")\n    ax.set_title(f\"Distribution of influences {title_extra}\")\n    return ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_influence_distribution_by_label","title":"plot_influence_distribution_by_label","text":"<pre><code>plot_influence_distribution_by_label(\n    influences: NDArray[float_], labels: NDArray[float_], title_extra: str = \"\"\n)\n</code></pre> <p>Plots the histogram of the influence that all samples in the training set have over a single sample index, separated by labels.</p> PARAMETER  DESCRIPTION <code>influences</code> <p>array of influences (training samples x test samples)</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>labels</code> <p>labels for the training set.</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>title_extra</code> <p>Additional text that will be appended to the title.</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_influence_distribution_by_label(\n    influences: NDArray[np.float_], labels: NDArray[np.float_], title_extra: str = \"\"\n):\n    \"\"\"Plots the histogram of the influence that all samples in the training set\n    have over a single sample index, separated by labels.\n\n    Args:\n       influences: array of influences (training samples x test samples)\n       labels: labels for the training set.\n       title_extra: Additional text that will be appended to the title.\n    \"\"\"\n    _, ax = plt.subplots()\n    unique_labels = np.unique(labels)\n    for label in unique_labels:\n        ax.hist(influences[labels == label], label=label, alpha=0.7)\n    ax.set_xlabel(\"Influence values\")\n    ax.set_ylabel(\"Number of samples\")\n    ax.set_title(f\"Distribution of influences {title_extra}\")\n    ax.legend()\n    plt.show()\n</code></pre>"},{"location":"api/pydvl/reporting/scores/","title":"Scores","text":""},{"location":"api/pydvl/reporting/scores/#pydvl.reporting.scores","title":"pydvl.reporting.scores","text":""},{"location":"api/pydvl/reporting/scores/#pydvl.reporting.scores.compute_removal_score","title":"compute_removal_score","text":"<pre><code>compute_removal_score(\n    u: Utility,\n    values: ValuationResult,\n    percentages: Union[NDArray[float_], Iterable[float]],\n    *,\n    remove_best: bool = False,\n    progress: bool = False\n) -&gt; Dict[float, float]\n</code></pre> <p>Fits model and computes score on the test set after incrementally removing a percentage of data points from the training set, based on their values.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>values</code> <p>Data values of data instances in the training set.</p> <p> TYPE: <code>ValuationResult</code> </p> <code>percentages</code> <p>Sequence of removal percentages.</p> <p> TYPE: <code>Union[NDArray[float_], Iterable[float]]</code> </p> <code>remove_best</code> <p>If True, removes data points in order of decreasing valuation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>progress</code> <p>If True, display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Dict[float, float]</code> <p>Dictionary that maps the percentages to their respective scores.</p> Source code in <code>src/pydvl/reporting/scores.py</code> <pre><code>def compute_removal_score(\n    u: Utility,\n    values: ValuationResult,\n    percentages: Union[NDArray[np.float_], Iterable[float]],\n    *,\n    remove_best: bool = False,\n    progress: bool = False,\n) -&gt; Dict[float, float]:\n    r\"\"\"Fits model and computes score on the test set after incrementally removing\n    a percentage of data points from the training set, based on their values.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        values: Data values of data instances in the training set.\n        percentages: Sequence of removal percentages.\n        remove_best: If True, removes data points in order of decreasing valuation.\n        progress: If True, display a progress bar.\n\n    Returns:\n        Dictionary that maps the percentages to their respective scores.\n    \"\"\"\n    # Sanity checks\n    if np.any([x &gt;= 1.0 or x &lt; 0.0 for x in percentages]):\n        raise ValueError(\"All percentages should be in the range [0.0, 1.0)\")\n\n    if len(values) != len(u.data.indices):\n        raise ValueError(\n            f\"The number of values, {len(values) }, should be equal to the number of data indices, {len(u.data.indices)}\"\n        )\n\n    scores = {}\n\n    # We sort in descending order if we want to remove the best values\n    values.sort(reverse=remove_best)\n\n    for pct in tqdm(percentages, disable=not progress, desc=\"Removal Scores\"):\n        n_removal = int(pct * len(u.data))\n        indices = values.indices[n_removal:]\n        score = u(indices)\n        scores[pct] = score\n    return scores\n</code></pre>"},{"location":"api/pydvl/utils/","title":"Utils","text":""},{"location":"api/pydvl/utils/#pydvl.utils","title":"pydvl.utils","text":""},{"location":"api/pydvl/utils/config/","title":"Config","text":""},{"location":"api/pydvl/utils/config/#pydvl.utils.config","title":"pydvl.utils.config","text":""},{"location":"api/pydvl/utils/config/#pydvl.utils.config.ParallelConfig","title":"ParallelConfig  <code>dataclass</code>","text":"<pre><code>ParallelConfig(\n    backend: Literal[\"joblib\", \"ray\"] = \"joblib\",\n    address: Optional[Union[str, Tuple[str, int]]] = None,\n    n_cpus_local: Optional[int] = None,\n    logging_level: Optional[int] = None,\n    wait_timeout: float = 1.0,\n)\n</code></pre> <p>Configuration for parallel computation backend.</p> PARAMETER  DESCRIPTION <code>backend</code> <p>Type of backend to use. Defaults to 'joblib'</p> <p> TYPE: <code>Literal['joblib', 'ray']</code> DEFAULT: <code>'joblib'</code> </p> <code>address</code> <p>(DEPRECATED) Address of existing remote or local cluster to use.</p> <p> TYPE: <code>Optional[Union[str, Tuple[str, int]]]</code> DEFAULT: <code>None</code> </p> <code>n_cpus_local</code> <p>(DEPRECATED) Number of CPUs to use when creating a local ray cluster. This has no effect when using an existing ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>logging_level</code> <p>(DEPRECATED) Logging level for the parallel backend's worker.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>wait_timeout</code> <p>(DEPRECATED) Timeout in seconds for waiting on futures.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p>"},{"location":"api/pydvl/utils/config/#pydvl.utils.config.CachedFuncConfig","title":"CachedFuncConfig  <code>dataclass</code>","text":"<pre><code>CachedFuncConfig(\n    hash_prefix: Optional[str] = None,\n    ignore_args: Collection[str] = list(),\n    time_threshold: float = 0.3,\n    allow_repeated_evaluations: bool = False,\n    rtol_stderr: float = 0.1,\n    min_repetitions: int = 3,\n)\n</code></pre> <p>Configuration for cached functions and methods, providing memoization of function calls.</p> <p>Instances of this class are typically used as arguments for the construction of a Utility.</p> PARAMETER  DESCRIPTION <code>hash_prefix</code> <p>Optional string prefix that be prepended to the cache key. This can be provided in order to guarantee cache reuse across runs.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ignore_args</code> <p>Do not take these keyword arguments into account when hashing the wrapped function for usage as key. This allows sharing the cache among different jobs for the same experiment run if the callable happens to have \"nuisance\" parameters like <code>job_id</code> which do not affect the result of the computation.</p> <p> TYPE: <code>Collection[str]</code> DEFAULT: <code>list()</code> </p> <code>time_threshold</code> <p>Computations taking less time than this many seconds are not cached. A value of 0 means that it will always cache results.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.3</code> </p> <code>allow_repeated_evaluations</code> <p>If <code>True</code>, repeated calls to a function with the same arguments will be allowed and outputs averaged until the running standard deviation of the mean stabilizes below <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>rtol_stderr</code> <p>relative tolerance for repeated evaluations. More precisely, memcached() will stop evaluating the function once the standard deviation of the mean is smaller than <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.1</code> </p> <code>min_repetitions</code> <p>minimum number of times that a function evaluation on the same arguments is repeated before returning cached values. Useful for stochastic functions only. If the model training is very noisy, set this number to higher values to reduce variance.</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p>"},{"location":"api/pydvl/utils/dataset/","title":"Dataset","text":""},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset","title":"pydvl.utils.dataset","text":"<p>This module contains convenience classes to handle data and groups thereof.</p> <p>Shapley and Least Core value computations require evaluation of a scoring function (the utility). This is typically the performance of the model on a test set (as an approximation to its true expected performance). It is therefore convenient to keep both the training data and the test data together to be passed around to methods in shapley and least_core. This is done with Dataset.</p> <p>This abstraction layer also seamlessly grouping data points together if one is interested in computing their value as a group, see GroupedDataset.</p> <p>Objects of both types are used to construct a Utility object.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset","title":"Dataset","text":"<pre><code>Dataset(\n    x_train: Union[NDArray, DataFrame],\n    y_train: Union[NDArray, DataFrame],\n    x_test: Union[NDArray, DataFrame],\n    y_test: Union[NDArray, DataFrame],\n    feature_names: Optional[Sequence[str]] = None,\n    target_names: Optional[Sequence[str]] = None,\n    data_names: Optional[Sequence[str]] = None,\n    description: Optional[str] = None,\n    is_multi_output: bool = False,\n)\n</code></pre> <p>A convenience class to handle datasets.</p> <p>It holds a dataset, split into training and test data, together with several labels on feature names, data point names and a description.</p> PARAMETER  DESCRIPTION <code>x_train</code> <p>training data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>y_train</code> <p>labels for training data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>x_test</code> <p>test data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>y_test</code> <p>labels for test data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>feature_names</code> <p>name of the features of input data</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>target_names</code> <p>names of the features of target data</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>names assigned to data points. For example, if the dataset is a time series, each entry can be a timestamp which can be referenced directly instead of using a row number.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>description</code> <p>A textual description of the dataset.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>is_multi_output</code> <p>set to <code>False</code> if labels are scalars, or to <code>True</code> if they are vectors of dimension &gt; 1.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def __init__(\n    self,\n    x_train: Union[NDArray, pd.DataFrame],\n    y_train: Union[NDArray, pd.DataFrame],\n    x_test: Union[NDArray, pd.DataFrame],\n    y_test: Union[NDArray, pd.DataFrame],\n    feature_names: Optional[Sequence[str]] = None,\n    target_names: Optional[Sequence[str]] = None,\n    data_names: Optional[Sequence[str]] = None,\n    description: Optional[str] = None,\n    # FIXME: use same parameter name as in check_X_y()\n    is_multi_output: bool = False,\n):\n    \"\"\"Constructs a Dataset from data and labels.\n\n    Args:\n        x_train: training data\n        y_train: labels for training data\n        x_test: test data\n        y_test: labels for test data\n        feature_names: name of the features of input data\n        target_names: names of the features of target data\n        data_names: names assigned to data points.\n            For example, if the dataset is a time series, each entry can be a\n            timestamp which can be referenced directly instead of using a row\n            number.\n        description: A textual description of the dataset.\n        is_multi_output: set to `False` if labels are scalars, or to\n            `True` if they are vectors of dimension &gt; 1.\n    \"\"\"\n    self.x_train, self.y_train = check_X_y(\n        x_train, y_train, multi_output=is_multi_output\n    )\n    self.x_test, self.y_test = check_X_y(\n        x_test, y_test, multi_output=is_multi_output\n    )\n\n    if x_train.shape[-1] != x_test.shape[-1]:\n        raise ValueError(\n            f\"Mismatching number of features: \"\n            f\"{x_train.shape[-1]} and {x_test.shape[-1]}\"\n        )\n    if x_train.shape[0] != y_train.shape[0]:\n        raise ValueError(\n            f\"Mismatching number of samples: \"\n            f\"{x_train.shape[-1]} and {x_test.shape[-1]}\"\n        )\n    if x_test.shape[0] != y_test.shape[0]:\n        raise ValueError(\n            f\"Mismatching number of samples: \"\n            f\"{x_test.shape[-1]} and {y_test.shape[-1]}\"\n        )\n\n    def make_names(s: str, a: np.ndarray) -&gt; List[str]:\n        n = a.shape[1] if len(a.shape) &gt; 1 else 1\n        return [f\"{s}{i:0{1 + int(np.log10(n))}d}\" for i in range(1, n + 1)]\n\n    self.feature_names = feature_names\n    self.target_names = target_names\n\n    if self.feature_names is None:\n        if isinstance(x_train, pd.DataFrame):\n            self.feature_names = x_train.columns.tolist()\n        else:\n            self.feature_names = make_names(\"x\", x_train)\n\n    if self.target_names is None:\n        if isinstance(y_train, pd.DataFrame):\n            self.target_names = y_train.columns.tolist()\n        else:\n            self.target_names = make_names(\"y\", y_train)\n\n    if len(self.x_train.shape) &gt; 1:\n        if (\n            len(self.feature_names) != self.x_train.shape[-1]\n            or len(self.feature_names) != self.x_test.shape[-1]\n        ):\n            raise ValueError(\"Mismatching number of features and names\")\n    if len(self.y_train.shape) &gt; 1:\n        if (\n            len(self.target_names) != self.y_train.shape[-1]\n            or len(self.target_names) != self.y_test.shape[-1]\n        ):\n            raise ValueError(\"Mismatching number of targets and names\")\n\n    self.description = description or \"No description\"\n    self._indices = np.arange(len(self.x_train), dtype=np.int_)\n    self._data_names = (\n        np.array(data_names, dtype=object)\n        if data_names is not None\n        else self._indices.astype(object)\n    )\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.indices","title":"indices  <code>property</code>","text":"<pre><code>indices: NDArray[int_]\n</code></pre> <p>Index of positions in data.x_train.</p> <p>Contiguous integers from 0 to len(Dataset).</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.data_names","title":"data_names  <code>property</code>","text":"<pre><code>data_names: NDArray[object_]\n</code></pre> <p>Names of each individual datapoint.</p> <p>Used for reporting Shapley values.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.dim","title":"dim  <code>property</code>","text":"<pre><code>dim: int\n</code></pre> <p>Returns the number of dimensions of a sample.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.get_training_data","title":"get_training_data","text":"<pre><code>get_training_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Given a set of indices, returns the training data that refer to those indices.</p> <p>This is used mainly by Utility to retrieve subsets of the data from indices. It is typically not needed in algorithms.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices that will be used to select points from the training data. If <code>None</code>, the entire training data will be returned.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>If <code>indices</code> is not <code>None</code>, the selected x and y arrays from the training data. Otherwise, the entire dataset.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_training_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Given a set of indices, returns the training data that refer to those\n    indices.\n\n    This is used mainly by [Utility][pydvl.utils.utility.Utility] to retrieve\n    subsets of the data from indices. It is typically **not needed in\n    algorithms**.\n\n    Args:\n        indices: Optional indices that will be used to select points from\n            the training data. If `None`, the entire training data will be\n            returned.\n\n    Returns:\n        If `indices` is not `None`, the selected x and y arrays from the\n            training data. Otherwise, the entire dataset.\n    \"\"\"\n    if indices is None:\n        return self.x_train, self.y_train\n    x = self.x_train[indices]\n    y = self.y_train[indices]\n    return x, y\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.get_test_data","title":"get_test_data","text":"<pre><code>get_test_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Returns the entire test set regardless of the passed indices.</p> <p>The passed indices will not be used because for data valuation we generally want to score the trained model on the entire test data.</p> <p>Additionally, the way this method is used in the Utility class, the passed indices will be those of the training data and would not work on the test data.</p> <p>There may be cases where it is desired to use parts of the test data. In those cases, it is recommended to inherit from Dataset and override get_test_data().</p> <p>For example, the following snippet shows how one could go about mapping the training data indices into test data indices inside get_test_data():</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n...    def get_test_data(self, indices=None):\n...        if indices is None:\n...            return self.x_test, self.y_test\n...        fraction = len(list(indices)) / len(self)\n...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n...        mapped_indices = np.unique(mapped_indices.astype(int))\n...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n...\n&gt;&gt;&gt; X = np.random.rand(100, 10)\n&gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n&gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n&gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n&gt;&gt;&gt; _ = dataset.get_training_data(indices)\n&gt;&gt;&gt; _ = dataset.get_test_data(indices)\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices into the test data. This argument is unused left for compatibility with get_training_data().</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>The entire test data.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_test_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Returns the entire test set regardless of the passed indices.\n\n    The passed indices will not be used because for data valuation\n    we generally want to score the trained model on the entire test data.\n\n    Additionally, the way this method is used in the\n    [Utility][pydvl.utils.utility.Utility] class, the passed indices will\n    be those of the training data and would not work on the test data.\n\n    There may be cases where it is desired to use parts of the test data.\n    In those cases, it is recommended to inherit from\n    [Dataset][pydvl.utils.dataset.Dataset] and override\n    [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data].\n\n    For example, the following snippet shows how one could go about\n    mapping the training data indices into test data indices\n    inside [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data]:\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n        ...    def get_test_data(self, indices=None):\n        ...        if indices is None:\n        ...            return self.x_test, self.y_test\n        ...        fraction = len(list(indices)) / len(self)\n        ...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n        ...        mapped_indices = np.unique(mapped_indices.astype(int))\n        ...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n        ...\n        &gt;&gt;&gt; X = np.random.rand(100, 10)\n        &gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n        &gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n        &gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n        &gt;&gt;&gt; _ = dataset.get_training_data(indices)\n        &gt;&gt;&gt; _ = dataset.get_test_data(indices)\n        ```\n\n    Args:\n        indices: Optional indices into the test data. This argument is\n            unused left for compatibility with\n            [get_training_data()][pydvl.utils.dataset.Dataset.get_training_data].\n\n    Returns:\n        The entire test data.\n    \"\"\"\n    return self.x_test, self.y_test\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.from_sklearn","title":"from_sklearn  <code>classmethod</code>","text":"<pre><code>from_sklearn(\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs\n) -&gt; Dataset\n</code></pre> <p>Constructs a Dataset object from a sklearn.utils.Bunch, as returned by the <code>load_*</code> functions in scikit-learn toy datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import load_boston\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n</code></pre> PARAMETER  DESCRIPTION <code>data</code> <p>scikit-learn Bunch object. The following attributes are supported:</p> <ul> <li><code>data</code>: covariates.</li> <li><code>target</code>: target variables (labels).</li> <li><code>feature_names</code> (optional): the feature names.</li> <li><code>target_names</code> (optional): the target names.</li> <li><code>DESCR</code> (optional): a description.</li> </ul> <p> TYPE: <code>Bunch</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the target variable as labels. Read more in scikit-learn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>is_multi_output</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the sklearn dataset</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_sklearn(\n    cls,\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs,\n) -&gt; \"Dataset\":\n    \"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from a\n    [sklearn.utils.Bunch][], as returned by the `load_*`\n    functions in [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html).\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import load_boston\n        &gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n        ```\n\n    Args:\n        data: scikit-learn Bunch object. The following attributes are supported:\n\n            - `data`: covariates.\n            - `target`: target variables (labels).\n            - `feature_names` (**optional**): the feature names.\n            - `target_names` (**optional**): the target names.\n            - `DESCR` (**optional**): a description.\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the target variable as labels. Read more in\n            [scikit-learn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `is_multi_output`.\n\n    Returns:\n        Object with the sklearn dataset\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\n    x_train, x_test, y_train, y_test = train_test_split(\n        data.data,\n        data.target,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=data.target if stratify_by_target else None,\n    )\n    return cls(\n        x_train,\n        y_train,\n        x_test,\n        y_test,\n        feature_names=data.get(\"feature_names\"),\n        target_names=data.get(\"target_names\"),\n        description=data.get(\"DESCR\"),\n        **kwargs,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.from_arrays","title":"from_arrays  <code>classmethod</code>","text":"<pre><code>from_arrays(\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs\n) -&gt; Dataset\n</code></pre> <p>Constructs a Dataset object from X and y numpy arrays  as returned by the <code>make_*</code> functions in sklearn generated datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import make_regression\n&gt;&gt;&gt; X, y = make_regression()\n&gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n</code></pre> PARAMETER  DESCRIPTION <code>X</code> <p>numpy array of shape (n_samples, n_features)</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>numpy array of shape (n_samples,)</p> <p> TYPE: <code>NDArray</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the y variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>feature_names</code> or <code>target_names</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the passed X and y arrays split across training and test sets.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_arrays(\n    cls,\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs,\n) -&gt; \"Dataset\":\n    \"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from X and y numpy arrays  as\n    returned by the `make_*` functions in [sklearn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import make_regression\n        &gt;&gt;&gt; X, y = make_regression()\n        &gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n        ```\n\n    Args:\n        X: numpy array of shape (n_samples, n_features)\n        y: numpy array of shape (n_samples,)\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified fashion,\n            using the y variable as labels. Read more in [sklearn's user\n            guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `feature_names`\n            or `target_names`.\n\n    Returns:\n        Object with the passed X and y arrays split across training and test sets.\n\n    !!! tip \"New in version 0.4.0\"\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\n    x_train, x_test, y_train, y_test = train_test_split(\n        X,\n        y,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=y if stratify_by_target else None,\n    )\n    return cls(x_train, y_train, x_test, y_test, **kwargs)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset","title":"GroupedDataset","text":"<pre><code>GroupedDataset(\n    x_train: NDArray,\n    y_train: NDArray,\n    x_test: NDArray,\n    y_test: NDArray,\n    data_groups: Sequence,\n    feature_names: Optional[Sequence[str]] = None,\n    target_names: Optional[Sequence[str]] = None,\n    group_names: Optional[Sequence[str]] = None,\n    description: Optional[str] = None,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>Dataset</code></p> <p>Used for calculating Shapley values of subsets of the data considered as logical units. For instance, one can group by value of a categorical feature, by bin into which a continuous feature falls, or by label.</p> PARAMETER  DESCRIPTION <code>x_train</code> <p>training data</p> <p> TYPE: <code>NDArray</code> </p> <code>y_train</code> <p>labels of training data</p> <p> TYPE: <code>NDArray</code> </p> <code>x_test</code> <p>test data</p> <p> TYPE: <code>NDArray</code> </p> <code>y_test</code> <p>labels of test data</p> <p> TYPE: <code>NDArray</code> </p> <code>data_groups</code> <p>Iterable of the same length as <code>x_train</code> containing a group label for each training data point. The label can be of any type, e.g. <code>str</code> or <code>int</code>. Data points with the same label will then be grouped by this object and considered as one for effects of valuation.</p> <p> TYPE: <code>Sequence</code> </p> <code>feature_names</code> <p>names of the covariates' features.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>target_names</code> <p>names of the labels or targets y</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>group_names</code> <p>names of the groups. If not provided, the labels from <code>data_groups</code> will be used.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>description</code> <p>A textual description of the dataset</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> <p>Changed in version 0.6.0</p> <p>Added <code>group_names</code> and forwarding of <code>kwargs</code></p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def __init__(\n    self,\n    x_train: NDArray,\n    y_train: NDArray,\n    x_test: NDArray,\n    y_test: NDArray,\n    data_groups: Sequence,\n    feature_names: Optional[Sequence[str]] = None,\n    target_names: Optional[Sequence[str]] = None,\n    group_names: Optional[Sequence[str]] = None,\n    description: Optional[str] = None,\n    **kwargs,\n):\n    \"\"\"Class for grouping datasets.\n\n    Used for calculating Shapley values of subsets of the data considered\n    as logical units. For instance, one can group by value of a categorical\n    feature, by bin into which a continuous feature falls, or by label.\n\n    Args:\n        x_train: training data\n        y_train: labels of training data\n        x_test: test data\n        y_test: labels of test data\n        data_groups: Iterable of the same length as `x_train` containing\n            a group label for each training data point. The label can be of any\n            type, e.g. `str` or `int`. Data points with the same label will\n            then be grouped by this object and considered as one for effects of\n            valuation.\n        feature_names: names of the covariates' features.\n        target_names: names of the labels or targets y\n        group_names: names of the groups. If not provided, the labels\n            from `data_groups` will be used.\n        description: A textual description of the dataset\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n\n    !!! tip \"Changed in version 0.6.0\"\n    Added `group_names` and forwarding of `kwargs`\n    \"\"\"\n    super().__init__(\n        x_train=x_train,\n        y_train=y_train,\n        x_test=x_test,\n        y_test=y_test,\n        feature_names=feature_names,\n        target_names=target_names,\n        description=description,\n        **kwargs,\n    )\n\n    if len(data_groups) != len(x_train):\n        raise ValueError(\n            f\"data_groups and x_train must have the same length.\"\n            f\"Instead got {len(data_groups)=} and {len(x_train)=}\"\n        )\n\n    self.groups: OrderedDict[Any, List[int]] = OrderedDict(\n        {k: [] for k in set(data_groups)}\n    )\n    for idx, group in enumerate(data_groups):\n        self.groups[group].append(idx)\n    self.group_items = list(self.groups.items())\n    self._indices = np.arange(len(self.groups.keys()))\n    self._data_names = (\n        np.array(group_names, dtype=object)\n        if group_names is not None\n        else np.array(list(self.groups.keys()), dtype=object)\n    )\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.dim","title":"dim  <code>property</code>","text":"<pre><code>dim: int\n</code></pre> <p>Returns the number of dimensions of a sample.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.indices","title":"indices  <code>property</code>","text":"<pre><code>indices\n</code></pre> <p>Indices of the groups.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.data_names","title":"data_names  <code>property</code>","text":"<pre><code>data_names\n</code></pre> <p>Names of the groups.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.get_test_data","title":"get_test_data","text":"<pre><code>get_test_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Returns the entire test set regardless of the passed indices.</p> <p>The passed indices will not be used because for data valuation we generally want to score the trained model on the entire test data.</p> <p>Additionally, the way this method is used in the Utility class, the passed indices will be those of the training data and would not work on the test data.</p> <p>There may be cases where it is desired to use parts of the test data. In those cases, it is recommended to inherit from Dataset and override get_test_data().</p> <p>For example, the following snippet shows how one could go about mapping the training data indices into test data indices inside get_test_data():</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n...    def get_test_data(self, indices=None):\n...        if indices is None:\n...            return self.x_test, self.y_test\n...        fraction = len(list(indices)) / len(self)\n...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n...        mapped_indices = np.unique(mapped_indices.astype(int))\n...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n...\n&gt;&gt;&gt; X = np.random.rand(100, 10)\n&gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n&gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n&gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n&gt;&gt;&gt; _ = dataset.get_training_data(indices)\n&gt;&gt;&gt; _ = dataset.get_test_data(indices)\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices into the test data. This argument is unused left for compatibility with get_training_data().</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>The entire test data.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_test_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Returns the entire test set regardless of the passed indices.\n\n    The passed indices will not be used because for data valuation\n    we generally want to score the trained model on the entire test data.\n\n    Additionally, the way this method is used in the\n    [Utility][pydvl.utils.utility.Utility] class, the passed indices will\n    be those of the training data and would not work on the test data.\n\n    There may be cases where it is desired to use parts of the test data.\n    In those cases, it is recommended to inherit from\n    [Dataset][pydvl.utils.dataset.Dataset] and override\n    [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data].\n\n    For example, the following snippet shows how one could go about\n    mapping the training data indices into test data indices\n    inside [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data]:\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n        ...    def get_test_data(self, indices=None):\n        ...        if indices is None:\n        ...            return self.x_test, self.y_test\n        ...        fraction = len(list(indices)) / len(self)\n        ...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n        ...        mapped_indices = np.unique(mapped_indices.astype(int))\n        ...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n        ...\n        &gt;&gt;&gt; X = np.random.rand(100, 10)\n        &gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n        &gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n        &gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n        &gt;&gt;&gt; _ = dataset.get_training_data(indices)\n        &gt;&gt;&gt; _ = dataset.get_test_data(indices)\n        ```\n\n    Args:\n        indices: Optional indices into the test data. This argument is\n            unused left for compatibility with\n            [get_training_data()][pydvl.utils.dataset.Dataset.get_training_data].\n\n    Returns:\n        The entire test data.\n    \"\"\"\n    return self.x_test, self.y_test\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.get_training_data","title":"get_training_data","text":"<pre><code>get_training_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Returns the data and labels of all samples in the given groups.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>group indices whose elements to return. If <code>None</code>, all data from all groups are returned.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>Tuple of training data x and labels y.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_training_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Returns the data and labels of all samples in the given groups.\n\n    Args:\n        indices: group indices whose elements to return. If `None`,\n            all data from all groups are returned.\n\n    Returns:\n        Tuple of training data x and labels y.\n    \"\"\"\n    if indices is None:\n        indices = self.indices\n    data_indices = [\n        idx for group_id in indices for idx in self.group_items[group_id][1]\n    ]\n    return super().get_training_data(data_indices)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_sklearn","title":"from_sklearn  <code>classmethod</code>","text":"<pre><code>from_sklearn(\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    data_groups: Optional[Sequence] = None,\n    **kwargs\n) -&gt; GroupedDataset\n</code></pre> <p>Constructs a GroupedDataset object from a sklearn.utils.Bunch as returned by the <code>load_*</code> functions in scikit-learn toy datasets and groups it.</p> Example <pre><code>&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; from pydvl.utils import GroupedDataset\n&gt;&gt;&gt; iris = load_iris()\n&gt;&gt;&gt; data_groups = iris.data[:, 0] // 0.5\n&gt;&gt;&gt; dataset = GroupedDataset.from_sklearn(iris, data_groups=data_groups)\n</code></pre> PARAMETER  DESCRIPTION <code>data</code> <p>scikit-learn Bunch object. The following attributes are supported:</p> <ul> <li><code>data</code>: covariates.</li> <li><code>target</code>: target variables (labels).</li> <li><code>feature_names</code> (optional): the feature names.</li> <li><code>target_names</code> (optional): the target names.</li> <li><code>DESCR</code> (optional): a description.</li> </ul> <p> TYPE: <code>Bunch</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the target variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>data_groups</code> <p>an array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Optional[Sequence]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>GroupedDataset</code> <p>Dataset with the selected sklearn data</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_sklearn(\n    cls,\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    data_groups: Optional[Sequence] = None,\n    **kwargs,\n) -&gt; \"GroupedDataset\":\n    \"\"\"Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from a\n    [sklearn.utils.Bunch][sklearn.utils.Bunch] as returned by the `load_*` functions in\n    [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html) and groups\n    it.\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from sklearn.datasets import load_iris\n        &gt;&gt;&gt; from pydvl.utils import GroupedDataset\n        &gt;&gt;&gt; iris = load_iris()\n        &gt;&gt;&gt; data_groups = iris.data[:, 0] // 0.5\n        &gt;&gt;&gt; dataset = GroupedDataset.from_sklearn(iris, data_groups=data_groups)\n        ```\n\n    Args:\n        data: scikit-learn Bunch object. The following attributes are supported:\n\n            - `data`: covariates.\n            - `target`: target variables (labels).\n            - `feature_names` (**optional**): the feature names.\n            - `target_names` (**optional**): the target names.\n            - `DESCR` (**optional**): a description.\n        train_size: size of the training dataset. Used in `train_test_split`.\n        random_state: seed for train / test split.\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the target variable as labels. Read more in\n            [sklearn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        data_groups: an array holding the group index or name for each\n            data point. The length of this array must be equal to the number of\n            data points in the dataset.\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n\n    Returns:\n        Dataset with the selected sklearn data\n    \"\"\"\n    if data_groups is None:\n        raise ValueError(\n            \"data_groups must be provided when constructing a GroupedDataset\"\n        )\n\n    x_train, x_test, y_train, y_test, data_groups_train, _ = train_test_split(\n        data.data,\n        data.target,\n        data_groups,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=data.target if stratify_by_target else None,\n    )\n\n    dataset = Dataset(\n        x_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test, **kwargs\n    )\n    return cls.from_dataset(dataset, data_groups_train)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_arrays","title":"from_arrays  <code>classmethod</code>","text":"<pre><code>from_arrays(\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    data_groups: Optional[Sequence] = None,\n    **kwargs\n) -&gt; Dataset\n</code></pre> <p>Constructs a GroupedDataset object from X and y numpy arrays as returned by the <code>make_*</code> functions in scikit-learn generated datasets.</p> Example <pre><code>&gt;&gt;&gt; from sklearn.datasets import make_classification\n&gt;&gt;&gt; from pydvl.utils import GroupedDataset\n&gt;&gt;&gt; X, y = make_classification(\n...     n_samples=100,\n...     n_features=4,\n...     n_informative=2,\n...     n_redundant=0,\n...     random_state=0,\n...     shuffle=False\n... )\n&gt;&gt;&gt; data_groups = X[:, 0] // 0.5\n&gt;&gt;&gt; dataset = GroupedDataset.from_arrays(X, y, data_groups=data_groups)\n</code></pre> PARAMETER  DESCRIPTION <code>X</code> <p>array of shape (n_samples, n_features)</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>array of shape (n_samples,)</p> <p> TYPE: <code>NDArray</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the y variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>data_groups</code> <p>an array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Optional[Sequence]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments that will be passed to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Dataset with the passed X and y arrays split across training and test sets.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_arrays(\n    cls,\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    data_groups: Optional[Sequence] = None,\n    **kwargs,\n) -&gt; \"Dataset\":\n    \"\"\"Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from X and y numpy arrays\n    as returned by the `make_*` functions in\n    [scikit-learn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from sklearn.datasets import make_classification\n        &gt;&gt;&gt; from pydvl.utils import GroupedDataset\n        &gt;&gt;&gt; X, y = make_classification(\n        ...     n_samples=100,\n        ...     n_features=4,\n        ...     n_informative=2,\n        ...     n_redundant=0,\n        ...     random_state=0,\n        ...     shuffle=False\n        ... )\n        &gt;&gt;&gt; data_groups = X[:, 0] // 0.5\n        &gt;&gt;&gt; dataset = GroupedDataset.from_arrays(X, y, data_groups=data_groups)\n        ```\n\n    Args:\n        X: array of shape (n_samples, n_features)\n        y: array of shape (n_samples,)\n        train_size: size of the training dataset. Used in `train_test_split`.\n        random_state: seed for train / test split.\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the y variable as labels. Read more in\n            [sklearn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        data_groups: an array holding the group index or name for each data\n            point. The length of this array must be equal to the number of\n            data points in the dataset.\n        kwargs: Additional keyword arguments that will be passed to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n\n    Returns:\n        Dataset with the passed X and y arrays split across training and\n            test sets.\n\n    !!! tip \"New in version 0.4.0\"\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\n    if data_groups is None:\n        raise ValueError(\n            \"data_groups must be provided when constructing a GroupedDataset\"\n        )\n    x_train, x_test, y_train, y_test, data_groups_train, _ = train_test_split(\n        X,\n        y,\n        data_groups,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=y if stratify_by_target else None,\n    )\n    dataset = Dataset(\n        x_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test, **kwargs\n    )\n    return cls.from_dataset(dataset, data_groups_train)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_dataset","title":"from_dataset  <code>classmethod</code>","text":"<pre><code>from_dataset(dataset: Dataset, data_groups: Sequence[Any]) -&gt; GroupedDataset\n</code></pre> <p>Creates a GroupedDataset object from the data a Dataset object and a mapping of data groups.</p> Example <pre><code>&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pydvl.utils import Dataset, GroupedDataset\n&gt;&gt;&gt; dataset = Dataset.from_arrays(\n...     X=np.asarray([[1, 2], [3, 4], [5, 6], [7, 8]]),\n...     y=np.asarray([0, 1, 0, 1]),\n... )\n&gt;&gt;&gt; dataset = GroupedDataset.from_dataset(dataset, data_groups=[0, 0, 1, 1])\n</code></pre> PARAMETER  DESCRIPTION <code>dataset</code> <p>The original data.</p> <p> TYPE: <code>Dataset</code> </p> <code>data_groups</code> <p>An array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Sequence[Any]</code> </p> RETURNS DESCRIPTION <code>GroupedDataset</code> <p>A GroupedDataset with the initial Dataset grouped by data_groups.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_dataset(\n    cls, dataset: Dataset, data_groups: Sequence[Any]\n) -&gt; \"GroupedDataset\":\n    \"\"\"Creates a [GroupedDataset][pydvl.utils.GroupedDataset] object from the data a\n    [Dataset][pydvl.utils.Dataset] object and a mapping of data groups.\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; from pydvl.utils import Dataset, GroupedDataset\n        &gt;&gt;&gt; dataset = Dataset.from_arrays(\n        ...     X=np.asarray([[1, 2], [3, 4], [5, 6], [7, 8]]),\n        ...     y=np.asarray([0, 1, 0, 1]),\n        ... )\n        &gt;&gt;&gt; dataset = GroupedDataset.from_dataset(dataset, data_groups=[0, 0, 1, 1])\n        ```\n\n    Args:\n        dataset: The original data.\n        data_groups: An array holding the group index or name for each data\n            point. The length of this array must be equal to the number of\n            data points in the dataset.\n\n    Returns:\n        A [GroupedDataset][pydvl.utils.GroupedDataset] with the initial\n            [Dataset][pydvl.utils.Dataset] grouped by data_groups.\n    \"\"\"\n    return cls(\n        x_train=dataset.x_train,\n        y_train=dataset.y_train,\n        x_test=dataset.x_test,\n        y_test=dataset.y_test,\n        data_groups=data_groups,\n        feature_names=dataset.feature_names,\n        target_names=dataset.target_names,\n        description=dataset.description,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/functional/","title":"Functional","text":""},{"location":"api/pydvl/utils/functional/#pydvl.utils.functional","title":"pydvl.utils.functional","text":"<p>Supporting utilities for manipulating arguments of functions.</p>"},{"location":"api/pydvl/utils/functional/#pydvl.utils.functional.free_arguments","title":"free_arguments","text":"<pre><code>free_arguments(fun: Union[Callable, partial]) -&gt; Set[str]\n</code></pre> <p>Computes the set of free arguments for a function or functools.partial object.</p> <p>All arguments of a function are considered free unless they are set by a partial. For example, if <code>f = partial(g, a=1)</code>, then <code>a</code> is not a free argument of <code>f</code>.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>A callable or a [partial object][].</p> <p> TYPE: <code>Union[Callable, partial]</code> </p> RETURNS DESCRIPTION <code>Set[str]</code> <p>The set of free arguments of <code>fun</code>.</p> <p>New in version 0.7.0</p> Source code in <code>src/pydvl/utils/functional.py</code> <pre><code>def free_arguments(fun: Union[Callable, partial]) -&gt; Set[str]:\n    \"\"\"Computes the set of free arguments for a function or\n    [functools.partial][] object.\n\n    All arguments of a function are considered free unless they are set by a\n    partial. For example, if `f = partial(g, a=1)`, then `a` is not a free\n    argument of `f`.\n\n    Args:\n        fun: A callable or a [partial object][].\n\n    Returns:\n        The set of free arguments of `fun`.\n\n    !!! tip \"New in version 0.7.0\"\n    \"\"\"\n    args_set_by_partial: Set[str] = set()\n\n    def _rec_unroll_partial_function_args(g: Union[Callable, partial]) -&gt; Callable:\n        \"\"\"Stores arguments and recursively call itself if `g` is a\n        [functools.partial][] object. In the end, returns the initially wrapped\n        function.\n\n        This handles the construct `partial(_accept_additional_argument, *args,\n        **kwargs)` that is used by `maybe_add_argument`.\n\n        Args:\n            g: A partial or a function to unroll.\n\n        Returns:\n            Initial wrapped function.\n        \"\"\"\n        nonlocal args_set_by_partial\n\n        if isinstance(g, partial) and g.func == _accept_additional_argument:\n            arg = g.keywords[\"arg\"]\n            if arg in args_set_by_partial:\n                args_set_by_partial.remove(arg)\n            return _rec_unroll_partial_function_args(g.keywords[\"fun\"])\n        elif isinstance(g, partial):\n            args_set_by_partial.update(g.keywords.keys())\n            args_set_by_partial.update(g.args)\n            return _rec_unroll_partial_function_args(g.func)\n        else:\n            return g\n\n    wrapped_fn = _rec_unroll_partial_function_args(fun)\n    sig = inspect.signature(wrapped_fn)\n    return args_set_by_partial | set(sig.parameters.keys())\n</code></pre>"},{"location":"api/pydvl/utils/functional/#pydvl.utils.functional.maybe_add_argument","title":"maybe_add_argument","text":"<pre><code>maybe_add_argument(fun: Callable, new_arg: str) -&gt; Callable\n</code></pre> <p>Wraps a function to accept the given keyword parameter if it doesn't already.</p> <p>If <code>fun</code> already takes a keyword parameter of name <code>new_arg</code>, then it is returned as is. Otherwise, a wrapper is returned which merely ignores the argument.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>The function to wrap</p> <p> TYPE: <code>Callable</code> </p> <code>new_arg</code> <p>The name of the argument that the new function will accept (and ignore).</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>A new function accepting one more keyword argument.</p> <p>Changed in version 0.7.0</p> <p>Ability to work with partials.</p> Source code in <code>src/pydvl/utils/functional.py</code> <pre><code>def maybe_add_argument(fun: Callable, new_arg: str) -&gt; Callable:\n    \"\"\"Wraps a function to accept the given keyword parameter if it doesn't\n    already.\n\n    If `fun` already takes a keyword parameter of name `new_arg`, then it is\n    returned as is. Otherwise, a wrapper is returned which merely ignores the\n    argument.\n\n    Args:\n        fun: The function to wrap\n        new_arg: The name of the argument that the new function will accept\n            (and ignore).\n\n    Returns:\n        A new function accepting one more keyword argument.\n\n    !!! tip \"Changed in version 0.7.0\"\n        Ability to work with partials.\n    \"\"\"\n    if new_arg in free_arguments(fun):\n        return fun\n\n    return partial(_accept_additional_argument, fun=fun, arg=new_arg)\n</code></pre>"},{"location":"api/pydvl/utils/numeric/","title":"Numeric","text":""},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric","title":"pydvl.utils.numeric","text":"<p>This module contains routines for numerical computations used across the library.</p>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.powerset","title":"powerset","text":"<pre><code>powerset(s: NDArray[T]) -&gt; Iterator[Collection[T]]\n</code></pre> <p>Returns an iterator for the power set of the argument.</p> <p>Subsets are generated in sequence by growing size. See  random_powerset() for random  sampling.</p> Example <pre><code>&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pydvl.utils.numeric import powerset\n&gt;&gt;&gt; list(powerset(np.array((1,2))))\n[(), (1,), (2,), (1, 2)]\n</code></pre> PARAMETER  DESCRIPTION <code>s</code> <p>The set to use</p> <p> TYPE: <code>NDArray[T]</code> </p> RETURNS DESCRIPTION <code>Iterator[Collection[T]]</code> <p>An iterator over all subsets of the set of indices <code>s</code>.</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def powerset(s: NDArray[T]) -&gt; Iterator[Collection[T]]:\n    \"\"\"Returns an iterator for the power set of the argument.\n\n     Subsets are generated in sequence by growing size. See\n     [random_powerset()][pydvl.utils.numeric.random_powerset] for random\n     sampling.\n\n    ??? Example\n        ``` pycon\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; from pydvl.utils.numeric import powerset\n        &gt;&gt;&gt; list(powerset(np.array((1,2))))\n        [(), (1,), (2,), (1, 2)]\n        ```\n\n    Args:\n         s: The set to use\n\n    Returns:\n        An iterator over all subsets of the set of indices `s`.\n    \"\"\"\n    return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1))\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.num_samples_permutation_hoeffding","title":"num_samples_permutation_hoeffding","text":"<pre><code>num_samples_permutation_hoeffding(\n    eps: float, delta: float, u_range: float\n) -&gt; int\n</code></pre> <p>Lower bound on the number of samples required for MonteCarlo Shapley to obtain an (\u03b5,\u03b4)-approximation.</p> <p>That is: with probability 1-\u03b4, the estimated value for one data point will be \u03b5-close to the true quantity, if at least this many permutations are sampled.</p> PARAMETER  DESCRIPTION <code>eps</code> <p>\u03b5 &gt; 0</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>0 &lt; \u03b4 &lt;= 1</p> <p> TYPE: <code>float</code> </p> <code>u_range</code> <p>Range of the Utility function</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>int</code> <p>Number of permutations required to guarantee \u03b5-correct Shapley values with probability 1-\u03b4</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def num_samples_permutation_hoeffding(eps: float, delta: float, u_range: float) -&gt; int:\n    \"\"\"Lower bound on the number of samples required for MonteCarlo Shapley to\n    obtain an (\u03b5,\u03b4)-approximation.\n\n    That is: with probability 1-\u03b4, the estimated value for one data point will\n    be \u03b5-close to the true quantity, if at least this many permutations are\n    sampled.\n\n    Args:\n        eps: \u03b5 &gt; 0\n        delta: 0 &lt; \u03b4 &lt;= 1\n        u_range: Range of the [Utility][pydvl.utils.utility.Utility] function\n\n    Returns:\n        Number of _permutations_ required to guarantee \u03b5-correct Shapley\n            values with probability 1-\u03b4\n    \"\"\"\n    return int(np.ceil(np.log(2 / delta) * 2 * u_range**2 / eps**2))\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_subset","title":"random_subset","text":"<pre><code>random_subset(\n    s: NDArray[T], q: float = 0.5, seed: Optional[Seed] = None\n) -&gt; NDArray[T]\n</code></pre> <p>Returns one subset at random from <code>s</code>.</p> PARAMETER  DESCRIPTION <code>s</code> <p>set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>q</code> <p>Sampling probability for elements. The default 0.5 yields a uniform distribution over the power set of s.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.5</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>The subset</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_subset(\n    s: NDArray[T], q: float = 0.5, seed: Optional[Seed] = None\n) -&gt; NDArray[T]:\n    \"\"\"Returns one subset at random from ``s``.\n\n    Args:\n        s: set to sample from\n        q: Sampling probability for elements. The default 0.5 yields a\n            uniform distribution over the power set of s.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n\n    Returns:\n        The subset\n    \"\"\"\n    rng = np.random.default_rng(seed)\n    selection = rng.uniform(size=len(s)) &gt; q\n    return s[selection]\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_powerset","title":"random_powerset","text":"<pre><code>random_powerset(\n    s: NDArray[T],\n    n_samples: Optional[int] = None,\n    q: float = 0.5,\n    seed: Optional[Seed] = None,\n) -&gt; Generator[NDArray[T], None, None]\n</code></pre> <p>Samples subsets from the power set of the argument, without pre-generating all subsets and in no order.</p> <p>See powerset if you wish to deterministically generate all subsets.</p> <p>To generate subsets, <code>len(s)</code> Bernoulli draws with probability <code>q</code> are drawn. The default value of <code>q = 0.5</code> provides a uniform distribution over the power set of <code>s</code>. Other choices can be used e.g. to implement owen_sampling_shapley.</p> PARAMETER  DESCRIPTION <code>s</code> <p>set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>n_samples</code> <p>if set, stop the generator after this many steps. Defaults to <code>np.iinfo(np.int32).max</code></p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>q</code> <p>Sampling probability for elements. The default 0.5 yields a uniform distribution over the power set of s.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.5</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Generator[NDArray[T], None, None]</code> <p>Samples from the power set of <code>s</code>.</p> RAISES DESCRIPTION <code>ValueError</code> <p>if the element sampling probability is not in [0,1]</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_powerset(\n    s: NDArray[T],\n    n_samples: Optional[int] = None,\n    q: float = 0.5,\n    seed: Optional[Seed] = None,\n) -&gt; Generator[NDArray[T], None, None]:\n    \"\"\"Samples subsets from the power set of the argument, without\n    pre-generating all subsets and in no order.\n\n    See [powerset][pydvl.utils.numeric.powerset] if you wish to deterministically generate all subsets.\n\n    To generate subsets, `len(s)` Bernoulli draws with probability `q` are\n    drawn. The default value of `q = 0.5` provides a uniform distribution over\n    the power set of `s`. Other choices can be used e.g. to implement\n    [owen_sampling_shapley][pydvl.value.shapley.owen.owen_sampling_shapley].\n\n    Args:\n        s: set to sample from\n        n_samples: if set, stop the generator after this many steps.\n            Defaults to `np.iinfo(np.int32).max`\n        q: Sampling probability for elements. The default 0.5 yields a\n            uniform distribution over the power set of s.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Samples from the power set of `s`.\n\n    Raises:\n        ValueError: if the element sampling probability is not in [0,1]\n\n    \"\"\"\n    if q &lt; 0 or q &gt; 1:\n        raise ValueError(\"Element sampling probability must be in [0,1]\")\n\n    rng = np.random.default_rng(seed)\n    total = 1\n    if n_samples is None:\n        n_samples = np.iinfo(np.int32).max\n    while total &lt;= n_samples:\n        yield random_subset(s, q, seed=rng)\n        total += 1\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_powerset_label_min","title":"random_powerset_label_min","text":"<pre><code>random_powerset_label_min(\n    s: NDArray[T],\n    labels: NDArray[int_],\n    min_elements_per_label: int = 1,\n    seed: Optional[Seed] = None,\n) -&gt; Generator[NDArray[T], None, None]\n</code></pre> <p>Draws random subsets from <code>s</code>, while ensuring that at least <code>min_elements_per_label</code> elements per label are included in the draw. It can be used for classification problems to ensure that a set contains information for all labels (or not if <code>min_elements_per_label=0</code>).</p> PARAMETER  DESCRIPTION <code>s</code> <p>Set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>labels</code> <p>Labels for the samples</p> <p> TYPE: <code>NDArray[int_]</code> </p> <code>min_elements_per_label</code> <p>Minimum number of elements for each label.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Generator[NDArray[T], None, None]</code> <p>Generated draw from the powerset of s with <code>min_elements_per_label</code> for each</p> <code>Generator[NDArray[T], None, None]</code> <p>label.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>s</code> and <code>labels</code> are of different length or <code>min_elements_per_label</code> is smaller than 0.</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_powerset_label_min(\n    s: NDArray[T],\n    labels: NDArray[np.int_],\n    min_elements_per_label: int = 1,\n    seed: Optional[Seed] = None,\n) -&gt; Generator[NDArray[T], None, None]:\n    \"\"\"Draws random subsets from `s`, while ensuring that at least\n    `min_elements_per_label` elements per label are included in the draw. It can be used\n    for classification problems to ensure that a set contains information for all labels\n    (or not if `min_elements_per_label=0`).\n\n    Args:\n        s: Set to sample from\n        labels: Labels for the samples\n        min_elements_per_label: Minimum number of elements for each label.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Generated draw from the powerset of s with `min_elements_per_label` for each\n        label.\n\n    Raises:\n        ValueError: If `s` and `labels` are of different length or\n            `min_elements_per_label` is smaller than 0.\n    \"\"\"\n    if len(labels) != len(s):\n        raise ValueError(\"Set and labels have to be of same size.\")\n\n    if min_elements_per_label &lt; 0:\n        raise ValueError(\n            f\"Parameter min_elements={min_elements_per_label} needs to be bigger or \"\n            f\"equal to 0.\"\n        )\n\n    rng = np.random.default_rng(seed)\n    unique_labels = np.unique(labels)\n\n    while True:\n        subsets: List[NDArray[T]] = []\n        for label in unique_labels:\n            label_indices = np.asarray(np.where(labels == label)[0])\n            subset_size = int(\n                rng.integers(\n                    min(min_elements_per_label, len(label_indices)),\n                    len(label_indices) + 1,\n                )\n            )\n            if subset_size &gt; 0:\n                subsets.append(\n                    random_subset_of_size(s[label_indices], subset_size, seed=rng)\n                )\n\n        if len(subsets) &gt; 0:\n            subset = np.concatenate(tuple(subsets))\n            rng.shuffle(subset)\n            yield subset\n        else:\n            yield np.array([], dtype=s.dtype)\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_subset_of_size","title":"random_subset_of_size","text":"<pre><code>random_subset_of_size(\n    s: NDArray[T], size: int, seed: Optional[Seed] = None\n) -&gt; NDArray[T]\n</code></pre> <p>Samples a random subset of given size uniformly from the powerset of <code>s</code>.</p> PARAMETER  DESCRIPTION <code>s</code> <p>Set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>size</code> <p>Size of the subset to generate</p> <p> TYPE: <code>int</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>The subset</p> <p>Raises     ValueError: If size &gt; len(s)</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_subset_of_size(\n    s: NDArray[T], size: int, seed: Optional[Seed] = None\n) -&gt; NDArray[T]:\n    \"\"\"Samples a random subset of given size uniformly from the powerset\n    of `s`.\n\n    Args:\n        s: Set to sample from\n        size: Size of the subset to generate\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        The subset\n\n    Raises\n        ValueError: If size &gt; len(s)\n    \"\"\"\n    if size &gt; len(s):\n        raise ValueError(\"Cannot sample subset larger than set\")\n    rng = np.random.default_rng(seed)\n    return rng.choice(s, size=size, replace=False)\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_matrix_with_condition_number","title":"random_matrix_with_condition_number","text":"<pre><code>random_matrix_with_condition_number(\n    n: int, condition_number: float, seed: Optional[Seed] = None\n) -&gt; NDArray\n</code></pre> <p>Constructs a square matrix with a given condition number.</p> <p>Taken from: https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py</p> <p>Also see: https://math.stackexchange.com/questions/1351616/condition-number-of-ata.</p> PARAMETER  DESCRIPTION <code>n</code> <p>size of the matrix</p> <p> TYPE: <code>int</code> </p> <code>condition_number</code> <p>duh</p> <p> TYPE: <code>float</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray</code> <p>An (n,n) matrix with the requested condition number.</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_matrix_with_condition_number(\n    n: int, condition_number: float, seed: Optional[Seed] = None\n) -&gt; NDArray:\n    \"\"\"Constructs a square matrix with a given condition number.\n\n    Taken from:\n    [https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py](\n    https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py)\n\n    Also see:\n    [https://math.stackexchange.com/questions/1351616/condition-number-of-ata](\n    https://math.stackexchange.com/questions/1351616/condition-number-of-ata).\n\n    Args:\n        n: size of the matrix\n        condition_number: duh\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        An (n,n) matrix with the requested condition number.\n    \"\"\"\n    if n &lt; 2:\n        raise ValueError(\"Matrix size must be at least 2\")\n\n    if condition_number &lt;= 1:\n        raise ValueError(\"Condition number must be greater than 1\")\n\n    rng = np.random.default_rng(seed)\n    log_condition_number = np.log(condition_number)\n    exp_vec = np.arange(\n        -log_condition_number / 4.0,\n        log_condition_number * (n + 1) / (4 * (n - 1)),\n        log_condition_number / (2.0 * (n - 1)),\n    )\n    exp_vec = exp_vec[:n]\n    s: np.ndarray = np.exp(exp_vec)\n    S = np.diag(s)\n    U, _ = np.linalg.qr((rng.uniform(size=(n, n)) - 5.0) * 200)\n    V, _ = np.linalg.qr((rng.uniform(size=(n, n)) - 5.0) * 200)\n    P: np.ndarray = U.dot(S).dot(V.T)\n    P = P.dot(P.T)\n    return P\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.running_moments","title":"running_moments","text":"<pre><code>running_moments(\n    previous_avg: float | NDArray[float_],\n    previous_variance: float | NDArray[float_],\n    count: int,\n    new_value: float | NDArray[float_],\n) -&gt; Tuple[float | NDArray[float_], float | NDArray[float_]]\n</code></pre> <p>Uses Welford's algorithm to calculate the running average and variance of  a set of numbers.</p> <p>See Welford's algorithm in wikipedia</p> <p>Warning</p> <p>This is not really using Welford's correction for numerical stability for the variance. (FIXME)</p> <p>Todo</p> <p>This could be generalised to arbitrary moments. See this paper</p> PARAMETER  DESCRIPTION <code>previous_avg</code> <p>average value at previous step</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> <code>previous_variance</code> <p>variance at previous step</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> <code>count</code> <p>number of points seen so far</p> <p> TYPE: <code>int</code> </p> <code>new_value</code> <p>new value in the series of numbers</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> RETURNS DESCRIPTION <code>Tuple[float | NDArray[float_], float | NDArray[float_]]</code> <p>new_average, new_variance, calculated with the new count</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def running_moments(\n    previous_avg: float | NDArray[np.float_],\n    previous_variance: float | NDArray[np.float_],\n    count: int,\n    new_value: float | NDArray[np.float_],\n) -&gt; Tuple[float | NDArray[np.float_], float | NDArray[np.float_]]:\n    \"\"\"Uses Welford's algorithm to calculate the running average and variance of\n     a set of numbers.\n\n    See [Welford's algorithm in wikipedia](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm)\n\n    !!! Warning\n        This is not really using Welford's correction for numerical stability\n        for the variance. (FIXME)\n\n    !!! Todo\n        This could be generalised to arbitrary moments. See [this paper](https://www.osti.gov/biblio/1028931)\n\n    Args:\n        previous_avg: average value at previous step\n        previous_variance: variance at previous step\n        count: number of points seen so far\n        new_value: new value in the series of numbers\n\n    Returns:\n        new_average, new_variance, calculated with the new count\n    \"\"\"\n    # broadcasted operations seem not to be supported by mypy, so we ignore the type\n    new_average = (new_value + count * previous_avg) / (count + 1)  # type: ignore\n    new_variance = previous_variance + (\n        (new_value - previous_avg) * (new_value - new_average) - previous_variance\n    ) / (count + 1)\n    return new_average, new_variance\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.top_k_value_accuracy","title":"top_k_value_accuracy","text":"<pre><code>top_k_value_accuracy(\n    y_true: NDArray[float_], y_pred: NDArray[float_], k: int = 3\n) -&gt; float\n</code></pre> <p>Computes the top-k accuracy for the estimated values by comparing indices of the highest k values.</p> PARAMETER  DESCRIPTION <code>y_true</code> <p>Exact/true value</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>y_pred</code> <p>Predicted/estimated value</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>k</code> <p>Number of the highest values taken into account</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p> RETURNS DESCRIPTION <code>float</code> <p>Accuracy</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def top_k_value_accuracy(\n    y_true: NDArray[np.float_], y_pred: NDArray[np.float_], k: int = 3\n) -&gt; float:\n    \"\"\"Computes the top-k accuracy for the estimated values by comparing indices\n    of the highest k values.\n\n    Args:\n        y_true: Exact/true value\n        y_pred: Predicted/estimated value\n        k: Number of the highest values taken into account\n\n    Returns:\n        Accuracy\n    \"\"\"\n    top_k_exact_values = np.argsort(y_true)[-k:]\n    top_k_pred_values = np.argsort(y_pred)[-k:]\n    top_k_accuracy = len(np.intersect1d(top_k_exact_values, top_k_pred_values)) / k\n    return top_k_accuracy\n</code></pre>"},{"location":"api/pydvl/utils/progress/","title":"Progress","text":""},{"location":"api/pydvl/utils/progress/#pydvl.utils.progress","title":"pydvl.utils.progress","text":""},{"location":"api/pydvl/utils/progress/#pydvl.utils.progress.repeat_indices","title":"repeat_indices","text":"<pre><code>repeat_indices(\n    indices: Collection[int],\n    result: ValuationResult,\n    done: StoppingCriterion,\n    **kwargs\n) -&gt; Iterator[int]\n</code></pre> <p>Helper function to cycle indefinitely over a collection of indices until the stopping criterion is satisfied while displaying progress.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>Collection of indices that will be cycled until done.</p> <p> TYPE: <code>Collection[int]</code> </p> <code>result</code> <p>Object containing the current results.</p> <p> TYPE: <code>ValuationResult</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>kwargs</code> <p>Keyword arguments passed to tqdm.</p> <p> DEFAULT: <code>{}</code> </p> Source code in <code>src/pydvl/utils/progress.py</code> <pre><code>def repeat_indices(\n    indices: Collection[int],\n    result: \"ValuationResult\",\n    done: \"StoppingCriterion\",\n    **kwargs,\n) -&gt; Iterator[int]:\n    \"\"\"Helper function to cycle indefinitely over a collection of indices\n    until the stopping criterion is satisfied while displaying progress.\n\n    Args:\n        indices: Collection of indices that will be cycled until done.\n        result: Object containing the current results.\n        done: Stopping criterion.\n        kwargs: Keyword arguments passed to tqdm.\n    \"\"\"\n    with tqdm(total=100, unit=\"%\", **kwargs) as pbar:\n        it = takewhile(lambda _: not done(result), cycle(indices))\n        for i in it:\n            yield i\n            pbar.update(100 * done.completion() - pbar.n)\n            pbar.refresh()\n</code></pre>"},{"location":"api/pydvl/utils/progress/#pydvl.utils.progress.log_duration","title":"log_duration","text":"<pre><code>log_duration(func)\n</code></pre> <p>Decorator to log execution time of a function</p> Source code in <code>src/pydvl/utils/progress.py</code> <pre><code>def log_duration(func):\n    \"\"\"\n    Decorator to log execution time of a function\n    \"\"\"\n\n    @wraps(func)\n    def wrapper_log_duration(*args, **kwargs):\n        func_name = func.__qualname__\n        logger.info(f\"Function '{func_name}' is starting.\")\n        start_time = time()\n        result = func(*args, **kwargs)\n        duration = time() - start_time\n        logger.info(f\"Function '{func_name}' completed. Duration: {duration:.2f} sec\")\n        return result\n\n    return wrapper_log_duration\n</code></pre>"},{"location":"api/pydvl/utils/score/","title":"Score","text":""},{"location":"api/pydvl/utils/score/#pydvl.utils.score","title":"pydvl.utils.score","text":"<p>This module provides a Scorer class that wraps scoring functions with additional information.</p> <p>Scorers are the fundamental building block of many data valuation methods. They are typically used by the Utility class to evaluate the quality of a model when trained on subsets of the training data.</p> <p>Scorers can be constructed in the same way as in scikit-learn: either from  known strings or from a callable. Greater values must be better. If they are not, a negated version can be used, see scikit-learn's make_scorer().</p> <p>Scorer provides additional information about the scoring function, like its range and default values, which can be used by some data valuation methods (like group_testing_shapley()) to estimate the number of samples required for a certain quality of approximation.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.squashed_r2","title":"squashed_r2  <code>module-attribute</code>","text":"<pre><code>squashed_r2 = compose_score(Scorer('r2'), _sigmoid, (0, 1), 'squashed r2')\n</code></pre> <p>A scorer that squashes the R\u00b2 score into the range [0, 1] using a sigmoid.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.squashed_variance","title":"squashed_variance  <code>module-attribute</code>","text":"<pre><code>squashed_variance = compose_score(\n    Scorer(\"explained_variance\"),\n    _sigmoid,\n    (0, 1),\n    \"squashed explained variance\",\n)\n</code></pre> <p>A scorer that squashes the explained variance score into the range [0, 1] using a sigmoid.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.ScorerCallable","title":"ScorerCallable","text":"<p>             Bases: <code>Protocol</code></p> <p>Signature for a scorer</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.Scorer","title":"Scorer","text":"<pre><code>Scorer(\n    scoring: Union[str, ScorerCallable],\n    default: float = np.nan,\n    range: Tuple = (-np.inf, np.inf),\n    name: Optional[str] = None,\n)\n</code></pre> <p>A scoring callable that takes a model, data, and labels and returns a scalar.</p> PARAMETER  DESCRIPTION <code>scoring</code> <p>Either a string or callable that can be passed to get_scorer.</p> <p> TYPE: <code>Union[str, ScorerCallable]</code> </p> <code>default</code> <p>score to be used when a model cannot be fit, e.g. when too little data is passed, or errors arise.</p> <p> TYPE: <code>float</code> DEFAULT: <code>nan</code> </p> <code>range</code> <p>numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation. If not provided, it can be read from the <code>scoring</code> object if it provides it, for instance if it was constructed with compose_score().</p> <p> TYPE: <code>Tuple</code> DEFAULT: <code>(-inf, inf)</code> </p> <code>name</code> <p>The name of the scorer. If not provided, the name of the function passed will be used.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <p>New in version 0.5.0</p> Source code in <code>src/pydvl/utils/score.py</code> <pre><code>def __init__(\n    self,\n    scoring: Union[str, ScorerCallable],\n    default: float = np.nan,\n    range: Tuple = (-np.inf, np.inf),\n    name: Optional[str] = None,\n):\n    if name is None and isinstance(scoring, str):\n        name = scoring\n    self._scorer = get_scorer(scoring)\n    self.default = default\n    # TODO: auto-fill from known scorers ?\n    self.range = np.array(range)\n    self._name = getattr(self._scorer, \"__name__\", name or \"scorer\")\n</code></pre>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.compose_score","title":"compose_score","text":"<pre><code>compose_score(\n    scorer: Scorer,\n    transformation: Callable[[float], float],\n    range: Tuple[float, float],\n    name: str,\n) -&gt; Scorer\n</code></pre> <p>Composes a scoring function with an arbitrary scalar transformation.</p> <p>Useful to squash unbounded scores into ranges manageable by data valuation methods.</p> <p>Example:</p> <pre><code>sigmoid = lambda x: 1/(1+np.exp(-x))\ncompose_score(Scorer(\"r2\"), sigmoid, range=(0,1), name=\"squashed r2\")\n</code></pre> PARAMETER  DESCRIPTION <code>scorer</code> <p>The object to be composed.</p> <p> TYPE: <code>Scorer</code> </p> <code>transformation</code> <p>A scalar transformation</p> <p> TYPE: <code>Callable[[float], float]</code> </p> <code>range</code> <p>The range of the transformation. This will be used e.g. by Utility for the range of the composed.</p> <p> TYPE: <code>Tuple[float, float]</code> </p> <code>name</code> <p>A string representation for the composition, for <code>str()</code>.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Scorer</code> <p>The composite Scorer.</p> Source code in <code>src/pydvl/utils/score.py</code> <pre><code>def compose_score(\n    scorer: Scorer,\n    transformation: Callable[[float], float],\n    range: Tuple[float, float],\n    name: str,\n) -&gt; Scorer:\n    \"\"\"Composes a scoring function with an arbitrary scalar transformation.\n\n    Useful to squash unbounded scores into ranges manageable by data valuation\n    methods.\n\n    Example:\n\n    ```python\n    sigmoid = lambda x: 1/(1+np.exp(-x))\n    compose_score(Scorer(\"r2\"), sigmoid, range=(0,1), name=\"squashed r2\")\n    ```\n\n    Args:\n        scorer: The object to be composed.\n        transformation: A scalar transformation\n        range: The range of the transformation. This will be used e.g. by\n            [Utility][pydvl.utils.utility.Utility] for the range of the composed.\n        name: A string representation for the composition, for `str()`.\n\n    Returns:\n        The composite [Scorer][pydvl.utils.score.Scorer].\n    \"\"\"\n\n    class CompositeScorer(Scorer):\n        def __call__(self, model: SupervisedModel, X: NDArray, y: NDArray) -&gt; float:\n            score = self._scorer(model=model, X=X, y=y)\n            return transformation(score)\n\n    return CompositeScorer(scorer, range=range, name=name)\n</code></pre>"},{"location":"api/pydvl/utils/status/","title":"Status","text":""},{"location":"api/pydvl/utils/status/#pydvl.utils.status","title":"pydvl.utils.status","text":""},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status","title":"Status","text":"<p>             Bases: <code>Enum</code></p> <p>Status of a computation.</p> <p>Statuses can be combined using bitwise or (<code>|</code>) and bitwise and (<code>&amp;</code>) to get the status of a combined computation. For example, if we have two computations, one that has converged and one that has failed, then the combined status is <code>Status.Converged | Status.Failed == Status.Converged</code>, but <code>Status.Converged &amp; Status.Failed == Status.Failed</code>.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--or","title":"OR","text":"<p>The result of bitwise or-ing two valuation statuses with <code>|</code> is given by the following table:</p> P C F P P C P C C C C F P C F <p>where P = Pending, C = Converged, F = Failed.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--and","title":"AND","text":"<p>The result of bitwise and-ing two valuation statuses with <code>&amp;</code> is given by the following table:</p> P C F P P P F C P C F F F F F <p>where P = Pending, C = Converged, F = Failed.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--not","title":"NOT","text":"<p>The result of bitwise negation of a Status with <code>~</code> is <code>Failed</code> if the status is <code>Converged</code>, or <code>Converged</code> otherwise:</p> <pre><code>~P == C, ~C == F, ~F == C\n</code></pre>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--boolean-casting","title":"Boolean casting","text":"<p>A Status evaluates to <code>True</code> iff it's <code>Converged</code> or <code>Failed</code>:</p> <pre><code>bool(Status.Pending) == False\nbool(Status.Converged) == True\nbool(Status.Failed) == True\n</code></pre> <p>Warning</p> <p>These truth values are inconsistent with the usual boolean operations. In particular the XOR of two instances of <code>Status</code> is not the same as the XOR of their boolean values.</p>"},{"location":"api/pydvl/utils/types/","title":"Types","text":""},{"location":"api/pydvl/utils/types/#pydvl.utils.types","title":"pydvl.utils.types","text":"<p>This module contains types, protocols, decorators and generic function transformations. Some of it probably belongs elsewhere.</p>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel","title":"SupervisedModel","text":"<p>             Bases: <code>Protocol</code></p> <p>This is the minimal Protocol that valuation methods require from models in order to work.</p> <p>All that is needed are the standard sklearn methods <code>fit()</code>, <code>predict()</code> and <code>score()</code>.</p>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel.fit","title":"fit","text":"<pre><code>fit(x: NDArray, y: NDArray)\n</code></pre> <p>Fit the model to the data</p> PARAMETER  DESCRIPTION <code>x</code> <p>Independent variables</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>Dependent variable</p> <p> TYPE: <code>NDArray</code> </p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def fit(self, x: NDArray, y: NDArray):\n    \"\"\"Fit the model to the data\n\n    Args:\n        x: Independent variables\n        y: Dependent variable\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel.predict","title":"predict","text":"<pre><code>predict(x: NDArray) -&gt; NDArray\n</code></pre> <p>Compute predictions for the input</p> PARAMETER  DESCRIPTION <code>x</code> <p>Independent variables for which to compute predictions</p> <p> TYPE: <code>NDArray</code> </p> RETURNS DESCRIPTION <code>NDArray</code> <p>Predictions for the input</p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def predict(self, x: NDArray) -&gt; NDArray:\n    \"\"\"Compute predictions for the input\n\n    Args:\n        x: Independent variables for which to compute predictions\n\n    Returns:\n        Predictions for the input\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel.score","title":"score","text":"<pre><code>score(x: NDArray, y: NDArray) -&gt; float\n</code></pre> <p>Compute the score of the model given test data</p> PARAMETER  DESCRIPTION <code>x</code> <p>Independent variables</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>Dependent variable</p> <p> TYPE: <code>NDArray</code> </p> RETURNS DESCRIPTION <code>float</code> <p>The score of the model on <code>(x, y)</code></p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def score(self, x: NDArray, y: NDArray) -&gt; float:\n    \"\"\"Compute the score of the model given test data\n\n    Args:\n        x: Independent variables\n        y: Dependent variable\n\n    Returns:\n        The score of the model on `(x, y)`\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.ensure_seed_sequence","title":"ensure_seed_sequence","text":"<pre><code>ensure_seed_sequence(\n    seed: Optional[Union[Seed, SeedSequence]] = None\n) -&gt; SeedSequence\n</code></pre> <p>If the passed seed is a SeedSequence object then it is returned as is. If it is a Generator the internal protected seed sequence from the generator gets extracted. Otherwise, a new SeedSequence object is created from the passed (optional) seed.</p> PARAMETER  DESCRIPTION <code>seed</code> <p>Either an int, a Generator object a SeedSequence object or None.</p> <p> TYPE: <code>Optional[Union[Seed, SeedSequence]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>SeedSequence</code> <p>A SeedSequence object.</p> <p>New in version 0.7.0</p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def ensure_seed_sequence(\n    seed: Optional[Union[Seed, SeedSequence]] = None\n) -&gt; SeedSequence:\n    \"\"\"\n    If the passed seed is a SeedSequence object then it is returned as is. If it is\n    a Generator the internal protected seed sequence from the generator gets extracted.\n    Otherwise, a new SeedSequence object is created from the passed (optional) seed.\n\n    Args:\n        seed: Either an int, a Generator object a SeedSequence object or None.\n\n    Returns:\n        A SeedSequence object.\n\n    !!! tip \"New in version 0.7.0\"\n    \"\"\"\n    if isinstance(seed, SeedSequence):\n        return seed\n    elif isinstance(seed, Generator):\n        return cast(SeedSequence, seed.bit_generator.seed_seq)  # type: ignore\n    else:\n        return SeedSequence(seed)\n</code></pre>"},{"location":"api/pydvl/utils/utility/","title":"Utility","text":""},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility","title":"pydvl.utils.utility","text":"<p>This module contains classes to manage and learn utility functions for the computation of values. Please see the documentation on Computing Data Values for more information.</p> <p>Utility holds information about model, data and scoring function (the latter being what one usually understands under utility in the general definition of Shapley value). It is automatically cached across machines when the cache is configured and it is enabled upon construction.</p> <p>DataUtilityLearning adds support for learning the scoring function to avoid repeated re-training of the model to compute the score.</p> <p>This module also contains derived <code>Utility</code> classes for toy games that are used for testing and for demonstration purposes.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility--references","title":"References","text":"<ol> <li> <p>Wang, T., Yang, Y. and Jia, R., 2021. Improving cooperative game theory-based data valuation via data utility learning. arXiv preprint arXiv:2107.06336.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility","title":"Utility","text":"<pre><code>Utility(\n    model: SupervisedModel,\n    data: Dataset,\n    scorer: Optional[Union[str, Scorer]] = None,\n    *,\n    default_score: float = 0.0,\n    score_range: Tuple[float, float] = (-np.inf, np.inf),\n    catch_errors: bool = True,\n    show_warnings: bool = False,\n    cache_backend: Optional[CacheBackend] = None,\n    cached_func_options: Optional[CachedFuncConfig] = None,\n    clone_before_fit: bool = True\n)\n</code></pre> <p>Convenience wrapper with configurable memoization of the scoring function.</p> <p>An instance of <code>Utility</code> holds the triple of model, dataset and scoring function which determines the value of data points. This is used for the computation of all game-theoretic values like Shapley values and the Least Core.</p> <p>The Utility expect the model to fulfill the SupervisedModel interface i.e. to have <code>fit()</code>, <code>predict()</code>, and <code>score()</code> methods.</p> <p>When calling the utility, the model will be cloned if it is a Sci-Kit Learn model, otherwise a copy is created using copy.deepcopy</p> <p>Since evaluating the scoring function requires retraining the model and that can be time-consuming, this class wraps it and caches the results of each execution. Caching is available both locally and across nodes, but must always be enabled for your project first, see the documentation and the module documentation.</p> ATTRIBUTE DESCRIPTION <code>model</code> <p>The supervised model.</p> <p> TYPE: <code>SupervisedModel</code> </p> <code>data</code> <p>An object containing the split data.</p> <p> TYPE: <code>Dataset</code> </p> <code>scorer</code> <p>A scoring function. If None, the <code>score()</code> method of the model will be used. See score for ways to create and compose scorers, in particular how to set default values and ranges.</p> <p> TYPE: <code>Scorer</code> </p> PARAMETER  DESCRIPTION <code>model</code> <p>Any supervised model. Typical choices can be found in the [sci-kit learn documentation][https://scikit-learn.org/stable/supervised_learning.html].</p> <p> TYPE: <code>SupervisedModel</code> </p> <code>data</code> <p>Dataset or GroupedDataset instance.</p> <p> TYPE: <code>Dataset</code> </p> <code>scorer</code> <p>A scoring object. If None, the <code>score()</code> method of the model will be used. See score for ways to create and compose scorers, in particular how to set default values and ranges. For convenience, a string can be passed, which will be used to construct a Scorer.</p> <p> TYPE: <code>Optional[Union[str, Scorer]]</code> DEFAULT: <code>None</code> </p> <code>default_score</code> <p>As a convenience when no <code>scorer</code> object is passed (where a default value can be provided), this argument also allows to set the default score for models that have not been fit, e.g. when too little data is passed, or errors arise.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>score_range</code> <p>As with <code>default_score</code>, this is a convenience argument for when no <code>scorer</code> argument is provided, to set the numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation.</p> <p> TYPE: <code>Tuple[float, float]</code> DEFAULT: <code>(-inf, inf)</code> </p> <code>catch_errors</code> <p>set to <code>True</code> to catch the errors when <code>fit()</code> fails. This could happen in several steps of the pipeline, e.g. when too little training data is passed, which happens often during Shapley value calculations. When this happens, the <code>default_score</code> is returned as a score and computation continues.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>show_warnings</code> <p>Set to <code>False</code> to suppress warnings thrown by <code>fit()</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>cache_backend</code> <p>Optional instance of CacheBackend used to wrap the _utility method of the Utility instance. By default, this is set to None and that means that the utility evaluations will not be cached.</p> <p> TYPE: <code>Optional[CacheBackend]</code> DEFAULT: <code>None</code> </p> <code>cached_func_options</code> <p>Optional configuration object for cached utility evaluation.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> <code>clone_before_fit</code> <p>If <code>True</code>, the model will be cloned before calling <code>fit()</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Utility, DataUtilityLearning, Dataset\n&gt;&gt;&gt; from sklearn.linear_model import LinearRegression, LogisticRegression\n&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_iris(), random_state=16)\n&gt;&gt;&gt; u = Utility(LogisticRegression(random_state=16), dataset)\n&gt;&gt;&gt; u(dataset.indices)\n0.9\n</code></pre> <p>With caching enabled:</p> <pre><code>&gt;&gt;&gt; from pydvl.utils import Utility, DataUtilityLearning, Dataset\n&gt;&gt;&gt; from pydvl.utils.caching.memory import InMemoryCacheBackend\n&gt;&gt;&gt; from sklearn.linear_model import LinearRegression, LogisticRegression\n&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_iris(), random_state=16)\n&gt;&gt;&gt; cache_backend = InMemoryCacheBackend()\n&gt;&gt;&gt; u = Utility(LogisticRegression(random_state=16), dataset, cache_backend=cache_backend)\n&gt;&gt;&gt; u(dataset.indices)\n0.9\n</code></pre> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(\n    self,\n    model: SupervisedModel,\n    data: Dataset,\n    scorer: Optional[Union[str, Scorer]] = None,\n    *,\n    default_score: float = 0.0,\n    score_range: Tuple[float, float] = (-np.inf, np.inf),\n    catch_errors: bool = True,\n    show_warnings: bool = False,\n    cache_backend: Optional[CacheBackend] = None,\n    cached_func_options: Optional[CachedFuncConfig] = None,\n    clone_before_fit: bool = True,\n):\n    self.model = self._clone_model(model)\n    self.data = data\n    if isinstance(scorer, str):\n        scorer = Scorer(scorer, default=default_score, range=score_range)\n    self.scorer = check_scoring(self.model, scorer)\n    self.default_score = scorer.default if scorer is not None else default_score\n    # TODO: auto-fill from known scorers ?\n    self.score_range = scorer.range if scorer is not None else np.array(score_range)\n    self.clone_before_fit = clone_before_fit\n    self.catch_errors = catch_errors\n    self.show_warnings = show_warnings\n    self.cache = cache_backend\n    if cached_func_options is None:\n        cached_func_options = CachedFuncConfig()\n    # TODO: Find a better way to do this.\n    if cached_func_options.hash_prefix is None:\n        # FIX: This does not handle reusing the same across runs.\n        cached_func_options.hash_prefix = str(hash((model, data, scorer)))\n    self.cached_func_options = cached_func_options\n    self._initialize_utility_wrapper()\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility.cache_stats","title":"cache_stats  <code>property</code>","text":"<pre><code>cache_stats: Optional[CacheStats]\n</code></pre> <p>Cache statistics are gathered when cache is enabled. See CacheStats for all fields returned.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility.__call__","title":"__call__","text":"<pre><code>__call__(indices: Iterable[int]) -&gt; float\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>a subset of valid indices for the <code>x_train</code> attribute of Dataset.</p> <p> TYPE: <code>Iterable[int]</code> </p> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __call__(self, indices: Iterable[int]) -&gt; float:\n    \"\"\"\n    Args:\n        indices: a subset of valid indices for the\n            `x_train` attribute of [Dataset][pydvl.utils.dataset.Dataset].\n    \"\"\"\n    utility: float = self._utility_wrapper(frozenset(indices))\n    return utility\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.DataUtilityLearning","title":"DataUtilityLearning","text":"<pre><code>DataUtilityLearning(u: Utility, training_budget: int, model: SupervisedModel)\n</code></pre> <p>Implementation of Data Utility Learning (Wang et al., 2022)<sup>1</sup>.</p> <p>This object wraps a Utility and delegates calls to it, up until a given budget (number of iterations). Every tuple of input and output (a so-called utility sample) is stored. Once the budget is exhausted, <code>DataUtilityLearning</code> fits the given model to the utility samples. Subsequent calls will use the learned model to predict the utility instead of delegating.</p> PARAMETER  DESCRIPTION <code>u</code> <p>The Utility to learn.</p> <p> TYPE: <code>Utility</code> </p> <code>training_budget</code> <p>Number of utility samples to collect before fitting the given model.</p> <p> TYPE: <code>int</code> </p> <code>model</code> <p>A supervised regression model</p> <p> TYPE: <code>SupervisedModel</code> </p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Utility, DataUtilityLearning, Dataset\n&gt;&gt;&gt; from sklearn.linear_model import LinearRegression, LogisticRegression\n&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_iris())\n&gt;&gt;&gt; u = Utility(LogisticRegression(), dataset)\n&gt;&gt;&gt; wrapped_u = DataUtilityLearning(u, 3, LinearRegression())\n... # First 3 calls will be computed normally\n&gt;&gt;&gt; for i in range(3):\n...     _ = wrapped_u((i,))\n&gt;&gt;&gt; wrapped_u((1, 2, 3)) # Subsequent calls will be computed using the fit model for DUL\n0.0\n</code></pre> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(\n    self, u: Utility, training_budget: int, model: SupervisedModel\n) -&gt; None:\n    self.utility = u\n    self.training_budget = training_budget\n    self.model = model\n    self._current_iteration = 0\n    self._is_model_fit = False\n    self._utility_samples: Dict[FrozenSet, Tuple[NDArray[np.bool_], float]] = {}\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.DataUtilityLearning.data","title":"data  <code>property</code>","text":"<pre><code>data: Dataset\n</code></pre> <p>Returns the wrapped utility's Dataset.</p>"},{"location":"api/pydvl/utils/caching/","title":"Caching","text":""},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching","title":"pydvl.utils.caching","text":"<p>This module provides caching of functions.</p> <p>PyDVL can cache (memoize) the computation of the utility function and speed up some computations for data valuation.</p> <p>Warning</p> <p>Function evaluations are cached with a key based on the function's signature and code. This can lead to undesired cache hits, see Cache reuse.</p> <p>Remember not to reuse utility objects for different datasets.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--configuration","title":"Configuration","text":"<p>Caching is disabled by default but can be enabled easily, see Setting up the cache. When enabled, it will be added to any callable used to construct a Utility (done with the wrap method of CacheBackend). Depending on the nature of the utility you might want to enable the computation of a running average of function values, see Usage with stochastic functions. You can see all configuration options under CachedFuncConfig.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--supported-backends","title":"Supported Backends","text":"<p>pyDVL supports 3 different caching backends:</p> <ul> <li>InMemoryCacheBackend:   an in-memory cache backend that uses a dictionary to store and retrieve   cached values. This is used to share cached values between threads   in a single process.</li> <li>DiskCacheBackend:   a disk-based cache backend that uses pickled values written to and read from disk.   This is used to share cached values between processes in a single machine.</li> <li> <p>MemcachedCacheBackend:   a Memcached-based cache backend that uses pickled values written to   and read from a Memcached server. This is used to share cached values   between processes across multiple machines.</p> <p>Info</p> <p>This specific backend requires optional dependencies not installed by default. See Extra dependencies for more information.</p> </li> </ul>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--usage-with-stochastic-functions","title":"Usage with stochastic functions","text":"<p>In addition to standard memoization, the wrapped functions can compute running average and standard error of repeated evaluations for the same input. This can be useful for stochastic functions with high variance (e.g. model training for small sample sizes), but drastically reduces the speed benefits of memoization.</p> <p>This behaviour can be activated with the option allow_repeated_evaluations.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--cache-reuse","title":"Cache reuse","text":"<p>When working directly with CachedFunc,  it is essential to only cache pure functions. If they have any kind of state, either internal or external (e.g. a closure over some data that may change), then the cache will fail to notice this and the same value will be returned.</p> <p>When a function is wrapped with CachedFunc for memoization, its signature (input and output names) and code are used as a key for the cache.</p> <p>If you are running experiments with the same Utility but different datasets, this will lead to evaluations of the utility on new data returning old values because utilities only use sample indices as arguments (so there is no way to tell the difference between '1' for dataset A and '1' for dataset 2 from the point of view of the cache). One solution is to empty the cache between runs by calling the <code>clear</code> method of the cache backend instance, but the preferred one is to use a different Utility object for each dataset.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--unexpected-cache-misses","title":"Unexpected cache misses","text":"<p>Because all arguments to a function are used as part of the key for the cache, sometimes one must exclude some of them. For example, If a function is going to run across multiple processes and some reporting arguments are added (like a <code>job_id</code> for logging purposes), these will be part of the signature and make the functions distinct to the eyes of the cache. This can be avoided with the use of ignore_args option in the configuration.</p>"},{"location":"api/pydvl/utils/caching/base/","title":"Base","text":""},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base","title":"pydvl.utils.caching.base","text":""},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheStats","title":"CacheStats  <code>dataclass</code>","text":"<pre><code>CacheStats(\n    sets: int = 0,\n    misses: int = 0,\n    hits: int = 0,\n    timeouts: int = 0,\n    errors: int = 0,\n    reconnects: int = 0,\n)\n</code></pre> <p>Class used to store statistics gathered by cached functions.</p> ATTRIBUTE DESCRIPTION <code>sets</code> <p>Number of times a value was set in the cache.</p> <p> TYPE: <code>int</code> </p> <code>misses</code> <p>Number of times a value was not found in the cache.</p> <p> TYPE: <code>int</code> </p> <code>hits</code> <p>Number of times a value was found in the cache.</p> <p> TYPE: <code>int</code> </p> <code>timeouts</code> <p>Number of times a timeout occurred.</p> <p> TYPE: <code>int</code> </p> <code>errors</code> <p>Number of times an error occurred.</p> <p> TYPE: <code>int</code> </p> <code>reconnects</code> <p>Number of times the client reconnected to the server.</p> <p> TYPE: <code>int</code> </p>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheResult","title":"CacheResult  <code>dataclass</code>","text":"<pre><code>CacheResult(value: float, count: int = 1, variance: float = 0.0)\n</code></pre> <p>A class used to store the cached result of a computation as well as count and variance when using repeated evaluation.</p> ATTRIBUTE DESCRIPTION <code>value</code> <p>Cached value.</p> <p> TYPE: <code>float</code> </p> <code>count</code> <p>Number of times this value has been computed.</p> <p> TYPE: <code>int</code> </p> <code>variance</code> <p>Variance associated with the cached value.</p> <p> TYPE: <code>float</code> </p>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend","title":"CacheBackend","text":"<pre><code>CacheBackend()\n</code></pre> <p>             Bases: <code>ABC</code></p> <p>Abstract base class for cache backends.</p> <p>Defines interface for cache access including wrapping callables, getting/setting results, clearing cache, and combining cache keys.</p> ATTRIBUTE DESCRIPTION <code>stats</code> <p>Cache statistics tracker.</p> <p> </p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def __init__(self) -&gt; None:\n    self.stats = CacheStats()\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend.wrap","title":"wrap","text":"<pre><code>wrap(\n    func: Callable, *, config: Optional[CachedFuncConfig] = None\n) -&gt; CachedFunc\n</code></pre> <p>Wraps a function to cache its results.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The function to wrap.</p> <p> TYPE: <code>Callable</code> </p> <code>config</code> <p>Optional caching options for the wrapped function.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>CachedFunc</code> <p>The wrapped cached function.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def wrap(\n    self,\n    func: Callable,\n    *,\n    config: Optional[CachedFuncConfig] = None,\n) -&gt; \"CachedFunc\":\n    \"\"\"Wraps a function to cache its results.\n\n    Args:\n        func: The function to wrap.\n        config: Optional caching options for the wrapped function.\n\n    Returns:\n        The wrapped cached function.\n    \"\"\"\n    return CachedFunc(\n        func,\n        cache_backend=self,\n        config=config,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend.get","title":"get  <code>abstractmethod</code>","text":"<pre><code>get(key: str) -&gt; Optional[CacheResult]\n</code></pre> <p>Abstract method to retrieve a cached result.</p> <p>Implemented by subclasses.</p> PARAMETER  DESCRIPTION <code>key</code> <p>The cache key.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Optional[CacheResult]</code> <p>The cached result or None if not found.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>@abstractmethod\ndef get(self, key: str) -&gt; Optional[CacheResult]:\n    \"\"\"Abstract method to retrieve a cached result.\n\n    Implemented by subclasses.\n\n    Args:\n        key: The cache key.\n\n    Returns:\n        The cached result or None if not found.\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend.set","title":"set  <code>abstractmethod</code>","text":"<pre><code>set(key: str, value: CacheResult) -&gt; None\n</code></pre> <p>Abstract method to set a cached result.</p> <p>Implemented by subclasses.</p> PARAMETER  DESCRIPTION <code>key</code> <p>The cache key.</p> <p> TYPE: <code>str</code> </p> <code>value</code> <p>The result to cache.</p> <p> TYPE: <code>CacheResult</code> </p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>@abstractmethod\ndef set(self, key: str, value: CacheResult) -&gt; None:\n    \"\"\"Abstract method to set a cached result.\n\n    Implemented by subclasses.\n\n    Args:\n        key: The cache key.\n        value: The result to cache.\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend.clear","title":"clear  <code>abstractmethod</code>","text":"<pre><code>clear() -&gt; None\n</code></pre> <p>Abstract method to clear the entire cache.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>@abstractmethod\ndef clear(self) -&gt; None:\n    \"\"\"Abstract method to clear the entire cache.\"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend.combine_hashes","title":"combine_hashes  <code>abstractmethod</code>","text":"<pre><code>combine_hashes(*args: str) -&gt; str\n</code></pre> <p>Abstract method to combine cache keys.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>@abstractmethod\ndef combine_hashes(self, *args: str) -&gt; str:\n    \"\"\"Abstract method to combine cache keys.\"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CachedFunc","title":"CachedFunc","text":"<pre><code>CachedFunc(\n    func: Callable[..., float],\n    *,\n    cache_backend: CacheBackend,\n    config: Optional[CachedFuncConfig] = None\n)\n</code></pre> <p>Caches callable function results with a provided cache backend.</p> <p>Wraps a callable function to cache its results using a provided an instance of a subclass of CacheBackend.</p> <p>This class is heavily inspired from that of joblib.memory.MemorizedFunc.</p> <p>This class caches calls to the wrapped callable by generating a hash key based on the wrapped callable's code, the arguments passed to it and the optional hash_prefix.</p> <p>Warning</p> <p>This class only works with hashable arguments to the wrapped callable.</p> PARAMETER  DESCRIPTION <code>func</code> <p>Callable to wrap.</p> <p> TYPE: <code>Callable[..., float]</code> </p> <code>cache_backend</code> <p>Instance of CacheBackendBase that handles setting and getting values.</p> <p> TYPE: <code>CacheBackend</code> </p> <code>config</code> <p>Configuration for wrapped function.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def __init__(\n    self,\n    func: Callable[..., float],\n    *,\n    cache_backend: CacheBackend,\n    config: Optional[CachedFuncConfig] = None,\n) -&gt; None:\n    self.func = func\n    self.cache_backend = cache_backend\n    if config is None:\n        config = CachedFuncConfig()\n    self.config = config\n\n    self.__doc__ = f\"A wrapper around {func.__name__}() with caching enabled.\\n\" + (\n        CachedFunc.__doc__ or \"\"\n    )\n    self.__name__ = f\"cached_{func.__name__}\"\n    path = list(reversed(func.__qualname__.split(\".\")))\n    patched = [f\"cached_{path[0]}\"] + path[1:]\n    self.__qualname__ = \".\".join(reversed(patched))\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CachedFunc.stats","title":"stats  <code>property</code>","text":"<pre><code>stats: CacheStats\n</code></pre> <p>Cache backend statistics.</p>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CachedFunc.__call__","title":"__call__","text":"<pre><code>__call__(*args, **kwargs) -&gt; float\n</code></pre> <p>Call the wrapped cached function.</p> <p>Executes the wrapped function, caching and returning the result.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def __call__(self, *args, **kwargs) -&gt; float:\n    \"\"\"Call the wrapped cached function.\n\n    Executes the wrapped function, caching and returning the result.\n    \"\"\"\n    return self._cached_call(args, kwargs)\n</code></pre>"},{"location":"api/pydvl/utils/caching/config/","title":"Config","text":""},{"location":"api/pydvl/utils/caching/config/#pydvl.utils.caching.config","title":"pydvl.utils.caching.config","text":""},{"location":"api/pydvl/utils/caching/config/#pydvl.utils.caching.config.CachedFuncConfig","title":"CachedFuncConfig  <code>dataclass</code>","text":"<pre><code>CachedFuncConfig(\n    hash_prefix: Optional[str] = None,\n    ignore_args: Collection[str] = list(),\n    time_threshold: float = 0.3,\n    allow_repeated_evaluations: bool = False,\n    rtol_stderr: float = 0.1,\n    min_repetitions: int = 3,\n)\n</code></pre> <p>Configuration for cached functions and methods, providing memoization of function calls.</p> <p>Instances of this class are typically used as arguments for the construction of a Utility.</p> PARAMETER  DESCRIPTION <code>hash_prefix</code> <p>Optional string prefix that be prepended to the cache key. This can be provided in order to guarantee cache reuse across runs.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ignore_args</code> <p>Do not take these keyword arguments into account when hashing the wrapped function for usage as key. This allows sharing the cache among different jobs for the same experiment run if the callable happens to have \"nuisance\" parameters like <code>job_id</code> which do not affect the result of the computation.</p> <p> TYPE: <code>Collection[str]</code> DEFAULT: <code>list()</code> </p> <code>time_threshold</code> <p>Computations taking less time than this many seconds are not cached. A value of 0 means that it will always cache results.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.3</code> </p> <code>allow_repeated_evaluations</code> <p>If <code>True</code>, repeated calls to a function with the same arguments will be allowed and outputs averaged until the running standard deviation of the mean stabilizes below <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>rtol_stderr</code> <p>relative tolerance for repeated evaluations. More precisely, memcached() will stop evaluating the function once the standard deviation of the mean is smaller than <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.1</code> </p> <code>min_repetitions</code> <p>minimum number of times that a function evaluation on the same arguments is repeated before returning cached values. Useful for stochastic functions only. If the model training is very noisy, set this number to higher values to reduce variance.</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p>"},{"location":"api/pydvl/utils/caching/disk/","title":"Disk","text":""},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk","title":"pydvl.utils.caching.disk","text":""},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend","title":"DiskCacheBackend","text":"<pre><code>DiskCacheBackend(cache_dir: Optional[Union[PathLike, str]] = None)\n</code></pre> <p>             Bases: <code>CacheBackend</code></p> <p>Disk cache backend that stores results in files.</p> <p>Implements the CacheBackend interface for a disk-based cache. Stores cache entries as pickled files on disk, keyed by cache key. This allows sharing evaluations across processes in a single node/computer.</p> PARAMETER  DESCRIPTION <code>cache_dir</code> <p>Base directory for cache storage.</p> <p> TYPE: <code>Optional[Union[PathLike, str]]</code> DEFAULT: <code>None</code> </p> ATTRIBUTE DESCRIPTION <code>cache_dir</code> <p>Base directory for cache storage.</p> <p> </p> Example <p>Basic usage: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.disk import DiskCacheBackend\n&gt;&gt;&gt; cache_backend = DiskCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; cache_backend.set(\"key\", value)\n&gt;&gt;&gt; cache_backend.get(\"key\")\n42\n</code></pre></p> <p>Callable wrapping: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.disk import DiskCacheBackend\n&gt;&gt;&gt; cache_backend = DiskCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; def foo(x: int):\n...     return x + 1\n...\n&gt;&gt;&gt; wrapped_foo = cache_backend.wrap(foo)\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n0\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n1\n</code></pre></p> PARAMETER  DESCRIPTION <code>cache_dir</code> <p>Base directory for cache storage. If not provided, this defaults to a newly created temporary directory.</p> <p> TYPE: <code>Optional[Union[PathLike, str]]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/utils/caching/disk.py</code> <pre><code>def __init__(\n    self,\n    cache_dir: Optional[Union[os.PathLike, str]] = None,\n) -&gt; None:\n    \"\"\"Initialize the disk cache backend.\n\n    Args:\n        cache_dir: Base directory for cache storage.\n            If not provided, this defaults to a newly created\n            temporary directory.\n    \"\"\"\n    super().__init__()\n    if cache_dir is None:\n        cache_dir = tempfile.mkdtemp(prefix=\"pydvl\")\n    self.cache_dir = Path(cache_dir)\n    self.cache_dir.mkdir(exist_ok=True, parents=True)\n</code></pre>"},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend.wrap","title":"wrap","text":"<pre><code>wrap(\n    func: Callable, *, config: Optional[CachedFuncConfig] = None\n) -&gt; CachedFunc\n</code></pre> <p>Wraps a function to cache its results.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The function to wrap.</p> <p> TYPE: <code>Callable</code> </p> <code>config</code> <p>Optional caching options for the wrapped function.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>CachedFunc</code> <p>The wrapped cached function.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def wrap(\n    self,\n    func: Callable,\n    *,\n    config: Optional[CachedFuncConfig] = None,\n) -&gt; \"CachedFunc\":\n    \"\"\"Wraps a function to cache its results.\n\n    Args:\n        func: The function to wrap.\n        config: Optional caching options for the wrapped function.\n\n    Returns:\n        The wrapped cached function.\n    \"\"\"\n    return CachedFunc(\n        func,\n        cache_backend=self,\n        config=config,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend.get","title":"get","text":"<pre><code>get(key: str) -&gt; Optional[Any]\n</code></pre> <p>Get a value from the cache.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Optional[Any]</code> <p>Cached value or None if not found.</p> Source code in <code>src/pydvl/utils/caching/disk.py</code> <pre><code>def get(self, key: str) -&gt; Optional[Any]:\n    \"\"\"Get a value from the cache.\n\n    Args:\n        key: Cache key.\n\n    Returns:\n        Cached value or None if not found.\n    \"\"\"\n    cache_file = self.cache_dir / key\n    if not cache_file.exists():\n        self.stats.misses += 1\n        return None\n    self.stats.hits += 1\n    with cache_file.open(\"rb\") as f:\n        return cloudpickle.load(f)\n</code></pre>"},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend.set","title":"set","text":"<pre><code>set(key: str, value: Any) -&gt; None\n</code></pre> <p>Set a value in the cache.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> <code>value</code> <p>Value to cache.</p> <p> TYPE: <code>Any</code> </p> Source code in <code>src/pydvl/utils/caching/disk.py</code> <pre><code>def set(self, key: str, value: Any) -&gt; None:\n    \"\"\"Set a value in the cache.\n\n    Args:\n        key: Cache key.\n        value: Value to cache.\n    \"\"\"\n    cache_file = self.cache_dir / key\n    self.stats.sets += 1\n    with cache_file.open(\"wb\") as f:\n        cloudpickle.dump(value, f, protocol=PICKLE_VERSION)\n</code></pre>"},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend.clear","title":"clear","text":"<pre><code>clear() -&gt; None\n</code></pre> <p>Deletes cache directory and recreates it.</p> Source code in <code>src/pydvl/utils/caching/disk.py</code> <pre><code>def clear(self) -&gt; None:\n    \"\"\"Deletes cache directory and recreates it.\"\"\"\n    shutil.rmtree(self.cache_dir)\n    self.cache_dir.mkdir(exist_ok=True, parents=True)\n</code></pre>"},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend.combine_hashes","title":"combine_hashes","text":"<pre><code>combine_hashes(*args: str) -&gt; str\n</code></pre> <p>Join cache key components.</p> Source code in <code>src/pydvl/utils/caching/disk.py</code> <pre><code>def combine_hashes(self, *args: str) -&gt; str:\n    \"\"\"Join cache key components.\"\"\"\n    return os.pathsep.join(args)\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/","title":"Memcached","text":""},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached","title":"pydvl.utils.caching.memcached","text":""},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedClientConfig","title":"MemcachedClientConfig  <code>dataclass</code>","text":"<pre><code>MemcachedClientConfig(\n    server: Tuple[str, int] = (\"localhost\", 11211),\n    connect_timeout: float = 1.0,\n    timeout: float = 1.0,\n    no_delay: bool = True,\n    serde: PickleSerde = PickleSerde(pickle_version=PICKLE_VERSION),\n)\n</code></pre> <p>Configuration of the memcached client.</p> PARAMETER  DESCRIPTION <code>server</code> <p>A tuple of (IP|domain name, port).</p> <p> TYPE: <code>Tuple[str, int]</code> DEFAULT: <code>('localhost', 11211)</code> </p> <code>connect_timeout</code> <p>How many seconds to wait before raising <code>ConnectionRefusedError</code> on failure to connect.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>timeout</code> <p>Duration in seconds to wait for send or recv calls on the socket connected to memcached.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>no_delay</code> <p>If True, set the <code>TCP_NODELAY</code> flag, which may help with performance in some cases.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>serde</code> <p>Serializer / Deserializer (\"serde\"). The default <code>PickleSerde</code> should work in most cases. See pymemcache.client.base.Client for details.</p> <p> TYPE: <code>PickleSerde</code> DEFAULT: <code>PickleSerde(pickle_version=PICKLE_VERSION)</code> </p>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend","title":"MemcachedCacheBackend","text":"<pre><code>MemcachedCacheBackend(config: MemcachedClientConfig = MemcachedClientConfig())\n</code></pre> <p>             Bases: <code>CacheBackend</code></p> <p>Memcached cache backend for the distributed caching of functions.</p> <p>Implements the CacheBackend interface for a memcached based cache. This allows sharing evaluations across processes and nodes in a cluster. You can run memcached as a service, locally or remotely, see the caching documentation.</p> PARAMETER  DESCRIPTION <code>config</code> <p>Memcached client configuration.</p> <p> TYPE: <code>MemcachedClientConfig</code> DEFAULT: <code>MemcachedClientConfig()</code> </p> ATTRIBUTE DESCRIPTION <code>config</code> <p>Memcached client configuration.</p> <p> </p> <code>client</code> <p>Memcached client instance.</p> <p> </p> Example <p>Basic usage: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.memcached import MemcachedCacheBackend\n&gt;&gt;&gt; cache_backend = MemcachedCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; cache_backend.set(\"key\", value)\n&gt;&gt;&gt; cache_backend.get(\"key\")\n42\n</code></pre></p> <p>Callable wrapping: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.memcached import MemcachedCacheBackend\n&gt;&gt;&gt; cache_backend = MemcachedCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; def foo(x: int):\n...     return x + 1\n...\n&gt;&gt;&gt; wrapped_foo = cache_backend.wrap(foo)\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n0\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n1\n</code></pre></p> PARAMETER  DESCRIPTION <code>config</code> <p>Memcached client configuration.</p> <p> TYPE: <code>MemcachedClientConfig</code> DEFAULT: <code>MemcachedClientConfig()</code> </p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def __init__(self, config: MemcachedClientConfig = MemcachedClientConfig()) -&gt; None:\n    \"\"\"Initialize memcached cache backend.\n\n    Args:\n        config: Memcached client configuration.\n    \"\"\"\n\n    super().__init__()\n    self.config = config\n    self.client = self._connect(self.config)\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.wrap","title":"wrap","text":"<pre><code>wrap(\n    func: Callable, *, config: Optional[CachedFuncConfig] = None\n) -&gt; CachedFunc\n</code></pre> <p>Wraps a function to cache its results.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The function to wrap.</p> <p> TYPE: <code>Callable</code> </p> <code>config</code> <p>Optional caching options for the wrapped function.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>CachedFunc</code> <p>The wrapped cached function.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def wrap(\n    self,\n    func: Callable,\n    *,\n    config: Optional[CachedFuncConfig] = None,\n) -&gt; \"CachedFunc\":\n    \"\"\"Wraps a function to cache its results.\n\n    Args:\n        func: The function to wrap.\n        config: Optional caching options for the wrapped function.\n\n    Returns:\n        The wrapped cached function.\n    \"\"\"\n    return CachedFunc(\n        func,\n        cache_backend=self,\n        config=config,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.get","title":"get","text":"<pre><code>get(key: str) -&gt; Optional[Any]\n</code></pre> <p>Get value from memcached.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Optional[Any]</code> <p>Cached value or None if not found or client disconnected.</p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def get(self, key: str) -&gt; Optional[Any]:\n    \"\"\"Get value from memcached.\n\n    Args:\n        key: Cache key.\n\n    Returns:\n        Cached value or None if not found or client disconnected.\n    \"\"\"\n    result = None\n    try:\n        result = self.client.get(key)\n    except socket.timeout as e:\n        self.stats.timeouts += 1\n        warnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\n    except OSError as e:\n        self.stats.errors += 1\n        warnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\n    except AttributeError as e:\n        # FIXME: this depends on _recv() failing on invalid sockets\n        # See pymemcache.base.py,\n        self.stats.reconnects += 1\n        warnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\n        self.client = self._connect(self.config)\n    if result is None:\n        self.stats.misses += 1\n    else:\n        self.stats.hits += 1\n    return result\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.set","title":"set","text":"<pre><code>set(key: str, value: Any) -&gt; None\n</code></pre> <p>Set value in memcached.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> <code>value</code> <p>Value to cache.</p> <p> TYPE: <code>Any</code> </p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def set(self, key: str, value: Any) -&gt; None:\n    \"\"\"Set value in memcached.\n\n    Args:\n        key: Cache key.\n        value: Value to cache.\n    \"\"\"\n    self.client.set(key, value, noreply=True)\n    self.stats.sets += 1\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.clear","title":"clear","text":"<pre><code>clear() -&gt; None\n</code></pre> <p>Flush all values from memcached.</p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def clear(self) -&gt; None:\n    \"\"\"Flush all values from memcached.\"\"\"\n    self.client.flush_all(noreply=True)\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.combine_hashes","title":"combine_hashes","text":"<pre><code>combine_hashes(*args: str) -&gt; str\n</code></pre> <p>Join cache key components for Memcached.</p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def combine_hashes(self, *args: str) -&gt; str:\n    \"\"\"Join cache key components for Memcached.\"\"\"\n    return \":\".join(args)\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.__getstate__","title":"__getstate__","text":"<pre><code>__getstate__() -&gt; Dict\n</code></pre> <p>Enables pickling after a socket has been opened to the memcached server, by removing the client from the stored data.</p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def __getstate__(self) -&gt; Dict:\n    \"\"\"Enables pickling after a socket has been opened to the\n    memcached server, by removing the client from the stored\n    data.\"\"\"\n    odict = self.__dict__.copy()\n    del odict[\"client\"]\n    return odict\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.__setstate__","title":"__setstate__","text":"<pre><code>__setstate__(d: Dict)\n</code></pre> <p>Restores a client connection after loading from a pickle.</p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def __setstate__(self, d: Dict):\n    \"\"\"Restores a client connection after loading from a pickle.\"\"\"\n    self.config = d[\"config\"]\n    self.stats = d[\"stats\"]\n    self.client = self._connect(self.config)\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/","title":"Memory","text":""},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory","title":"pydvl.utils.caching.memory","text":""},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend","title":"InMemoryCacheBackend","text":"<pre><code>InMemoryCacheBackend()\n</code></pre> <p>             Bases: <code>CacheBackend</code></p> <p>In-memory cache backend that stores results in a dictionary.</p> <p>Implements the CacheBackend interface for an in-memory-based cache. Stores cache entries as values in a dictionary, keyed by cache key. This allows sharing evaluations across threads in a single process.</p> <p>The implementation is not thread-safe.</p> ATTRIBUTE DESCRIPTION <code>cached_values</code> <p>Dictionary used to store cached values.</p> <p> TYPE: <code>Dict[str, Any]</code> </p> Example <p>Basic usage: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.memory import InMemoryCacheBackend\n&gt;&gt;&gt; cache_backend = InMemoryCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; cache_backend.set(\"key\", value)\n&gt;&gt;&gt; cache_backend.get(\"key\")\n42\n</code></pre></p> <p>Callable wrapping: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.memory import InMemoryCacheBackend\n&gt;&gt;&gt; cache_backend = InMemoryCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; def foo(x: int):\n...     return x + 1\n...\n&gt;&gt;&gt; wrapped_foo = cache_backend.wrap(foo)\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n0\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n1\n</code></pre></p> Source code in <code>src/pydvl/utils/caching/memory.py</code> <pre><code>def __init__(self) -&gt; None:\n    \"\"\"Initialize the in-memory cache backend.\"\"\"\n    super().__init__()\n    self.cached_values: Dict[str, Any] = {}\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend.wrap","title":"wrap","text":"<pre><code>wrap(\n    func: Callable, *, config: Optional[CachedFuncConfig] = None\n) -&gt; CachedFunc\n</code></pre> <p>Wraps a function to cache its results.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The function to wrap.</p> <p> TYPE: <code>Callable</code> </p> <code>config</code> <p>Optional caching options for the wrapped function.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>CachedFunc</code> <p>The wrapped cached function.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def wrap(\n    self,\n    func: Callable,\n    *,\n    config: Optional[CachedFuncConfig] = None,\n) -&gt; \"CachedFunc\":\n    \"\"\"Wraps a function to cache its results.\n\n    Args:\n        func: The function to wrap.\n        config: Optional caching options for the wrapped function.\n\n    Returns:\n        The wrapped cached function.\n    \"\"\"\n    return CachedFunc(\n        func,\n        cache_backend=self,\n        config=config,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend.get","title":"get","text":"<pre><code>get(key: str) -&gt; Optional[Any]\n</code></pre> <p>Get a value from the cache.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Optional[Any]</code> <p>Cached value or None if not found.</p> Source code in <code>src/pydvl/utils/caching/memory.py</code> <pre><code>def get(self, key: str) -&gt; Optional[Any]:\n    \"\"\"Get a value from the cache.\n\n    Args:\n        key: Cache key.\n\n    Returns:\n        Cached value or None if not found.\n    \"\"\"\n    value = self.cached_values.get(key, None)\n    if value is not None:\n        self.stats.hits += 1\n    else:\n        self.stats.misses += 1\n    return value\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend.set","title":"set","text":"<pre><code>set(key: str, value: Any) -&gt; None\n</code></pre> <p>Set a value in the cache.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> <code>value</code> <p>Value to cache.</p> <p> TYPE: <code>Any</code> </p> Source code in <code>src/pydvl/utils/caching/memory.py</code> <pre><code>def set(self, key: str, value: Any) -&gt; None:\n    \"\"\"Set a value in the cache.\n\n    Args:\n        key: Cache key.\n        value: Value to cache.\n    \"\"\"\n    self.cached_values[key] = value\n    self.stats.sets += 1\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend.clear","title":"clear","text":"<pre><code>clear() -&gt; None\n</code></pre> <p>Deletes cache dictionary and recreates it.</p> Source code in <code>src/pydvl/utils/caching/memory.py</code> <pre><code>def clear(self) -&gt; None:\n    \"\"\"Deletes cache dictionary and recreates it.\"\"\"\n    del self.cached_values\n    self.cached_values = {}\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend.combine_hashes","title":"combine_hashes","text":"<pre><code>combine_hashes(*args: str) -&gt; str\n</code></pre> <p>Join cache key components.</p> Source code in <code>src/pydvl/utils/caching/memory.py</code> <pre><code>def combine_hashes(self, *args: str) -&gt; str:\n    \"\"\"Join cache key components.\"\"\"\n    return os.pathsep.join(args)\n</code></pre>"},{"location":"api/pydvl/value/","title":"Value","text":""},{"location":"api/pydvl/value/#pydvl.value","title":"pydvl.value","text":"<p>This module implements algorithms for the exact and approximate computation of values and semi-values.</p> <p>See Data valuation for an introduction to the concepts and methods implemented here.</p>"},{"location":"api/pydvl/value/games/","title":"Games","text":""},{"location":"api/pydvl/value/games/#pydvl.value.games","title":"pydvl.value.games","text":"<p>This module provides several predefined games and, depending on the game, the corresponding Shapley values, Least Core values or both of them, for benchmarking purposes.</p>"},{"location":"api/pydvl/value/games/#pydvl.value.games--references","title":"References","text":"<ol> <li> <p>Castro, J., G\u00f3mez, D. and Tejada,   J., 2009. Polynomial calculation of the Shapley value based on   sampling.   Computers &amp; Operations Research, 36(5), pp.1726-1730.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset","title":"DummyGameDataset","text":"<pre><code>DummyGameDataset(n_players: int, description: Optional[str] = None)\n</code></pre> <p>             Bases: <code>Dataset</code></p> <p>Dummy game dataset.</p> <p>Initializes a dummy game dataset with n_players and an optional description.</p> <p>This class is used internally inside the Game class.</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> </p> <code>description</code> <p>Optional description of the dataset.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int, description: Optional[str] = None) -&gt; None:\n    x = np.arange(0, n_players, 1).reshape(-1, 1)\n    nil = np.zeros_like(x)\n    super().__init__(\n        x,\n        nil.copy(),\n        nil.copy(),\n        nil.copy(),\n        feature_names=[\"x\"],\n        target_names=[\"y\"],\n        description=description,\n    )\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.indices","title":"indices  <code>property</code>","text":"<pre><code>indices: NDArray[int_]\n</code></pre> <p>Index of positions in data.x_train.</p> <p>Contiguous integers from 0 to len(Dataset).</p>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.data_names","title":"data_names  <code>property</code>","text":"<pre><code>data_names: NDArray[object_]\n</code></pre> <p>Names of each individual datapoint.</p> <p>Used for reporting Shapley values.</p>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.dim","title":"dim  <code>property</code>","text":"<pre><code>dim: int\n</code></pre> <p>Returns the number of dimensions of a sample.</p>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.get_training_data","title":"get_training_data","text":"<pre><code>get_training_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Given a set of indices, returns the training data that refer to those indices.</p> <p>This is used mainly by Utility to retrieve subsets of the data from indices. It is typically not needed in algorithms.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices that will be used to select points from the training data. If <code>None</code>, the entire training data will be returned.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>If <code>indices</code> is not <code>None</code>, the selected x and y arrays from the training data. Otherwise, the entire dataset.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_training_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Given a set of indices, returns the training data that refer to those\n    indices.\n\n    This is used mainly by [Utility][pydvl.utils.utility.Utility] to retrieve\n    subsets of the data from indices. It is typically **not needed in\n    algorithms**.\n\n    Args:\n        indices: Optional indices that will be used to select points from\n            the training data. If `None`, the entire training data will be\n            returned.\n\n    Returns:\n        If `indices` is not `None`, the selected x and y arrays from the\n            training data. Otherwise, the entire dataset.\n    \"\"\"\n    if indices is None:\n        return self.x_train, self.y_train\n    x = self.x_train[indices]\n    y = self.y_train[indices]\n    return x, y\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.from_sklearn","title":"from_sklearn  <code>classmethod</code>","text":"<pre><code>from_sklearn(\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs\n) -&gt; Dataset\n</code></pre> <p>Constructs a Dataset object from a sklearn.utils.Bunch, as returned by the <code>load_*</code> functions in scikit-learn toy datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import load_boston\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n</code></pre> PARAMETER  DESCRIPTION <code>data</code> <p>scikit-learn Bunch object. The following attributes are supported:</p> <ul> <li><code>data</code>: covariates.</li> <li><code>target</code>: target variables (labels).</li> <li><code>feature_names</code> (optional): the feature names.</li> <li><code>target_names</code> (optional): the target names.</li> <li><code>DESCR</code> (optional): a description.</li> </ul> <p> TYPE: <code>Bunch</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the target variable as labels. Read more in scikit-learn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>is_multi_output</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the sklearn dataset</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_sklearn(\n    cls,\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs,\n) -&gt; \"Dataset\":\n    \"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from a\n    [sklearn.utils.Bunch][], as returned by the `load_*`\n    functions in [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html).\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import load_boston\n        &gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n        ```\n\n    Args:\n        data: scikit-learn Bunch object. The following attributes are supported:\n\n            - `data`: covariates.\n            - `target`: target variables (labels).\n            - `feature_names` (**optional**): the feature names.\n            - `target_names` (**optional**): the target names.\n            - `DESCR` (**optional**): a description.\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the target variable as labels. Read more in\n            [scikit-learn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `is_multi_output`.\n\n    Returns:\n        Object with the sklearn dataset\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\n    x_train, x_test, y_train, y_test = train_test_split(\n        data.data,\n        data.target,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=data.target if stratify_by_target else None,\n    )\n    return cls(\n        x_train,\n        y_train,\n        x_test,\n        y_test,\n        feature_names=data.get(\"feature_names\"),\n        target_names=data.get(\"target_names\"),\n        description=data.get(\"DESCR\"),\n        **kwargs,\n    )\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.from_arrays","title":"from_arrays  <code>classmethod</code>","text":"<pre><code>from_arrays(\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs\n) -&gt; Dataset\n</code></pre> <p>Constructs a Dataset object from X and y numpy arrays  as returned by the <code>make_*</code> functions in sklearn generated datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import make_regression\n&gt;&gt;&gt; X, y = make_regression()\n&gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n</code></pre> PARAMETER  DESCRIPTION <code>X</code> <p>numpy array of shape (n_samples, n_features)</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>numpy array of shape (n_samples,)</p> <p> TYPE: <code>NDArray</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the y variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>feature_names</code> or <code>target_names</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the passed X and y arrays split across training and test sets.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_arrays(\n    cls,\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs,\n) -&gt; \"Dataset\":\n    \"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from X and y numpy arrays  as\n    returned by the `make_*` functions in [sklearn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import make_regression\n        &gt;&gt;&gt; X, y = make_regression()\n        &gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n        ```\n\n    Args:\n        X: numpy array of shape (n_samples, n_features)\n        y: numpy array of shape (n_samples,)\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified fashion,\n            using the y variable as labels. Read more in [sklearn's user\n            guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `feature_names`\n            or `target_names`.\n\n    Returns:\n        Object with the passed X and y arrays split across training and test sets.\n\n    !!! tip \"New in version 0.4.0\"\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\n    x_train, x_test, y_train, y_test = train_test_split(\n        X,\n        y,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=y if stratify_by_target else None,\n    )\n    return cls(x_train, y_train, x_test, y_test, **kwargs)\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.get_test_data","title":"get_test_data","text":"<pre><code>get_test_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Returns the subsets of the train set instead of the test set.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>Indices into the training data.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>Subset of the train data.</p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def get_test_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Returns the subsets of the train set instead of the test set.\n\n    Args:\n        indices: Indices into the training data.\n\n    Returns:\n        Subset of the train data.\n    \"\"\"\n    if indices is None:\n        return self.x_train, self.y_train\n    x = self.x_train[indices]\n    y = self.y_train[indices]\n    return x, y\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyModel","title":"DummyModel","text":"<pre><code>DummyModel()\n</code></pre> <p>             Bases: <code>SupervisedModel</code></p> <p>Dummy model class.</p> <p>A dummy supervised model used for testing purposes only.</p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self) -&gt; None:\n    pass\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.Game","title":"Game","text":"<pre><code>Game(\n    n_players: int,\n    score_range: Tuple[float, float] = (-np.inf, np.inf),\n    description: Optional[str] = None,\n)\n</code></pre> <p>             Bases: <code>ABC</code></p> <p>Base class for games</p> <p>Any Game subclass has to implement the abstract <code>_score</code> method to assign a score to each coalition/subset and at least one of <code>shapley_values</code>, <code>least_core_values</code>.</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> </p> <code>score_range</code> <p>Minimum and maximum values of the <code>_score</code> method.</p> <p> TYPE: <code>Tuple[float, float]</code> DEFAULT: <code>(-inf, inf)</code> </p> <code>description</code> <p>Optional string description of the dummy dataset that will be created.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> ATTRIBUTE DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> </p> <code>data</code> <p>Dummy dataset object.</p> <p> </p> <code>u</code> <p>Utility object with a dummy model and dataset.</p> <p> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(\n    self,\n    n_players: int,\n    score_range: Tuple[float, float] = (-np.inf, np.inf),\n    description: Optional[str] = None,\n):\n    self.n_players = n_players\n    self.data = DummyGameDataset(self.n_players, description)\n    self.u = Utility(\n        DummyModel(),\n        self.data,\n        scorer=Scorer(self._score, range=score_range),\n        catch_errors=False,\n        show_warnings=True,\n    )\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.SymmetricVotingGame","title":"SymmetricVotingGame","text":"<pre><code>SymmetricVotingGame(n_players: int)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>A symmetric voting game defined in (Castro et al., 2009)<sup>1</sup> Section 4.1</p> <p>For this game the utility of a coalition is 1 if its cardinality is greater than num_samples/2, or 0 otherwise.</p> \\[{ v(S) = \\left\\{\\begin{array}{ll} 1, &amp; \\text{ if} \\quad \\mid S \\mid &gt; \\frac{N}{2} \\\\ 0, &amp; \\text{ otherwise} \\end{array}\\right. }\\] PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int) -&gt; None:\n    if n_players % 2 != 0:\n        raise ValueError(\"n_players must be an even number.\")\n    description = \"Dummy data for the symmetric voting game in Castro et al. 2009\"\n    super().__init__(\n        n_players,\n        score_range=(0, 1),\n        description=description,\n    )\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.AsymmetricVotingGame","title":"AsymmetricVotingGame","text":"<pre><code>AsymmetricVotingGame(n_players: int = 51)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>An asymmetric voting game defined in (Castro et al., 2009)<sup>1</sup> Section 4.2.</p> <p>For this game the player set is \\(N = \\{1,\\dots,51\\}\\) and the utility of a coalition is given by:</p> \\[{ v(S) = \\left\\{\\begin{array}{ll} 1, &amp; \\text{ if} \\quad \\sum\\limits_{i \\in S} w_i &gt; \\sum\\limits_{j \\in N}\\frac{w_j}{2} \\\\ 0, &amp; \\text{ otherwise} \\end{array}\\right. }\\] <p>where \\(w = [w_1,\\dots, w_{51}]\\) is a list of weights associated with each player.</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> DEFAULT: <code>51</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int = 51) -&gt; None:\n    if n_players != 51:\n        raise ValueError(\n            f\"{self.__class__.__name__} only supports n_players=51 but got {n_players=}.\"\n        )\n    description = \"Dummy data for the asymmetric voting game in Castro et al. 2009\"\n    super().__init__(\n        n_players,\n        score_range=(0, 1),\n        description=description,\n    )\n\n    ranges = [\n        range(0, 1),\n        range(1, 2),\n        range(2, 3),\n        range(3, 5),\n        range(5, 6),\n        range(6, 7),\n        range(7, 9),\n        range(9, 10),\n        range(10, 12),\n        range(12, 15),\n        range(15, 16),\n        range(16, 20),\n        range(20, 24),\n        range(24, 26),\n        range(26, 30),\n        range(30, 34),\n        range(34, 35),\n        range(35, 44),\n        range(44, 51),\n    ]\n\n    ranges_weights = [\n        45,\n        41,\n        27,\n        26,\n        25,\n        21,\n        17,\n        14,\n        13,\n        12,\n        11,\n        10,\n        9,\n        8,\n        7,\n        6,\n        5,\n        4,\n        3,\n    ]\n    ranges_values = [\n        \"0.08831\",\n        \"0.07973\",\n        \"0.05096\",\n        \"0.04898\",\n        \"0.047\",\n        \"0.03917\",\n        \"0.03147\",\n        \"0.02577\",\n        \"0.02388\",\n        \"0.022\",\n        \"0.02013\",\n        \"0.01827\",\n        \"0.01641\",\n        \"0.01456\",\n        \"0.01272\",\n        \"0.01088\",\n        \"0.009053\",\n        \"0.00723\",\n        \"0.005412\",\n    ]\n\n    self.weight_table = np.zeros(self.n_players)\n    exact_values = np.zeros(self.n_players)\n    for r, w, v in zip(ranges, ranges_weights, ranges_values):\n        self.weight_table[r] = w\n        exact_values[r] = v\n\n    self.exact_values = exact_values\n    self.threshold = np.sum(self.weight_table) / 2\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.ShoesGame","title":"ShoesGame","text":"<pre><code>ShoesGame(left: int, right: int)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>A shoes game defined in (Castro et al., 2009)<sup>1</sup>.</p> <p>In this game, some players have a left shoe and others a right shoe. Single shoes have a worth of zero while pairs have a worth of 1.</p> <p>The payoff of a coalition \\(S\\) is:</p> \\[{ v(S) = \\min( \\mid S \\cap L \\mid, \\mid S \\cap R \\mid ) }\\] <p>Where \\(L\\), respectively \\(R\\), is the set of players with left shoes, respectively right shoes.</p> PARAMETER  DESCRIPTION <code>left</code> <p>Number of players with a left shoe.</p> <p> TYPE: <code>int</code> </p> <code>right</code> <p>Number of players with a right shoe.</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, left: int, right: int) -&gt; None:\n    self.left = left\n    self.right = right\n    n_players = self.left + self.right\n    description = \"Dummy data for the shoe game in Castro et al. 2009\"\n    max_score = n_players // 2\n    super().__init__(n_players, score_range=(0, max_score), description=description)\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.AirportGame","title":"AirportGame","text":"<pre><code>AirportGame(n_players: int = 100)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>An airport game defined in (Castro et al., 2009)<sup>1</sup> Section 4.3</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> DEFAULT: <code>100</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int = 100) -&gt; None:\n    if n_players != 100:\n        raise ValueError(\n            f\"{self.__class__.__name__} only supports n_players=100 but got {n_players=}.\"\n        )\n    description = \"A dummy dataset for the airport game in Castro et al. 2009\"\n    super().__init__(n_players, score_range=(0, 100), description=description)\n    ranges = [\n        range(0, 8),\n        range(8, 20),\n        range(20, 26),\n        range(26, 40),\n        range(40, 48),\n        range(48, 57),\n        range(57, 70),\n        range(70, 80),\n        range(80, 90),\n        range(90, 100),\n    ]\n    exact = [\n        0.01,\n        0.020869565,\n        0.033369565,\n        0.046883079,\n        0.063549745,\n        0.082780515,\n        0.106036329,\n        0.139369662,\n        0.189369662,\n        0.289369662,\n    ]\n    c = list(range(1, 10))\n    score_table = np.zeros(100)\n    exact_values = np.zeros(100)\n\n    for r, v in zip(ranges, exact):\n        score_table[r] = c\n        exact_values[r] = v\n\n    self.exact_values = exact_values\n    self.score_table = score_table\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.MinimumSpanningTreeGame","title":"MinimumSpanningTreeGame","text":"<pre><code>MinimumSpanningTreeGame(n_players: int = 100)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>A minimum spanning tree game defined in (Castro et al., 2009)<sup>1</sup>.</p> <p>Let \\(G = (N \\cup \\{0\\},E)\\) be a valued graph where \\(N = \\{1,\\dots,100\\}\\), and the cost associated to an edge \\((i, j)\\) is:</p> \\[{ c_{ij} = \\left\\{\\begin{array}{lll} 1, &amp; \\text{ if} &amp; i = j + 1 \\text{ or } i = j - 1 \\\\ &amp; &amp; \\text{ or } (i = 1 \\text{ and } j = 100) \\text{ or } (i = 100 \\text{ and } j = 1) \\\\ 101, &amp; \\text{ if} &amp; i = 0 \\text{ or } j = 0 \\\\ \\infty, &amp; \\text{ otherwise} \\end{array}\\right. }\\] <p>A minimum spanning tree game \\((N, c)\\) is a cost game, where for a given coalition \\(S \\subset N\\), \\(v(S)\\) is the sum of the edge cost of the minimum spanning tree, i.e. \\(v(S)\\) = Minimum Spanning Tree of the graph \\(G|_{S\\cup\\{0\\}}\\), which is the partial graph restricted to the players \\(S\\) and the source node \\(0\\).</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> DEFAULT: <code>100</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int = 100) -&gt; None:\n    if n_players != 100:\n        raise ValueError(\n            f\"{self.__class__.__name__} only supports n_players=100 but got {n_players=}.\"\n        )\n    description = (\n        \"A dummy dataset for the minimum spanning tree game in Castro et al. 2009\"\n    )\n    super().__init__(n_players, score_range=(0, np.inf), description=description)\n\n    graph = np.zeros(shape=(self.n_players, self.n_players))\n\n    for i in range(self.n_players):\n        for j in range(self.n_players):\n            if (\n                i == j + 1\n                or i == j - 1\n                or (i == 1 and j == self.n_players - 1)\n                or (i == self.n_players - 1 and j == 1)\n            ):\n                graph[i, j] = 1\n            elif i == 0 or j == 0:\n                graph[i, j] = 0\n            else:\n                graph[i, j] = np.inf\n    assert np.all(graph == graph.T)\n\n    self.graph = graph\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.MinerGame","title":"MinerGame","text":"<pre><code>MinerGame(n_players: int)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>Consider a group of n miners, who have discovered large bars of gold.</p> <p>If two miners can carry one piece of gold, then the payoff of a coalition \\(S\\) is:</p> \\[{ v(S) = \\left\\{\\begin{array}{lll} \\mid S \\mid / 2, &amp; \\text{ if} &amp; \\mid S \\mid \\text{ is even} \\\\ ( \\mid S \\mid - 1)/2, &amp; \\text{ otherwise} \\end{array}\\right. }\\] <p>If there are more than two miners and there is an even number of miners, then the core consists of the single payoff where each miner gets 1/2.</p> <p>If there is an odd number of miners, then the core is empty.</p> <p>Taken from Wikipedia</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of miners that participate in the game.</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int) -&gt; None:\n    if n_players &lt;= 2:\n        raise ValueError(f\"n_players, {n_players}, should be &gt; 2\")\n    description = \"Dummy data for Miner Game taken from https://en.wikipedia.org/wiki/Core_(game_theory)\"\n    super().__init__(\n        n_players,\n        score_range=(0, n_players // 2),\n        description=description,\n    )\n</code></pre>"},{"location":"api/pydvl/value/result/","title":"Result","text":""},{"location":"api/pydvl/value/result/#pydvl.value.result","title":"pydvl.value.result","text":"<p>This module collects types and methods for the inspection of the results of valuation algorithms.</p> <p>The most important class is ValuationResult, which provides access to raw values, as well as convenient behaviour as a <code>Sequence</code> with extended indexing and updating abilities, and conversion to pandas DataFrames.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result--operating-on-results","title":"Operating on results","text":"<p>Results can be added together with the standard <code>+</code> operator. Because values are typically running averages of iterative algorithms, addition behaves like a weighted average of the two results, with the weights being the number of updates in each result: adding two results is the same as generating one result with the mean of the values of the two results as values. The variances are updated accordingly. See ValuationResult for details.</p> <p>Results can also be sorted by value, variance or number of updates, see sort(). The arrays of ValuationResult.values, ValuationResult.variances, ValuationResult.counts, ValuationResult.indices, ValuationResult.names are sorted in the same way.</p> <p>Indexing and slicing of results is supported and ValueItem objects are returned. These objects can be compared with the usual operators, which take only the ValueItem.value into account.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result--creating-result-objects","title":"Creating result objects","text":"<p>The most commonly used factory method is ValuationResult.zeros(), which creates a result object with all values, variances and counts set to zero. ValuationResult.empty() creates an empty result object, which can be used as a starting point for adding results together. Empty results are discarded when added to other results. Finally, ValuationResult.from_random() samples random values uniformly.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValueItem","title":"ValueItem  <code>dataclass</code>","text":"<pre><code>ValueItem(\n    index: IndexT,\n    name: NameT,\n    value: float,\n    variance: Optional[float],\n    count: Optional[int],\n)\n</code></pre> <p>             Bases: <code>Generic[IndexT, NameT]</code></p> <p>The result of a value computation for one datum.</p> <p><code>ValueItems</code> can be compared with the usual operators, forming a total order. Comparisons take only the <code>value</code> into account.</p> <p>Todo</p> <p>Maybe have a mode of comparing similar to <code>np.isclose</code>, or taking the <code>variance</code> into account.</p> ATTRIBUTE DESCRIPTION <code>index</code> <p>Index of the sample with this value in the original Dataset</p> <p> TYPE: <code>IndexT</code> </p> <code>name</code> <p>Name of the sample if it was provided. Otherwise, <code>str(index)</code></p> <p> TYPE: <code>NameT</code> </p> <code>value</code> <p>The value</p> <p> TYPE: <code>float</code> </p> <code>variance</code> <p>Variance of the value if it was computed with an approximate method</p> <p> TYPE: <code>Optional[float]</code> </p> <code>count</code> <p>Number of updates for this value</p> <p> TYPE: <code>Optional[int]</code> </p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValueItem.stderr","title":"stderr  <code>property</code>","text":"<pre><code>stderr: Optional[float]\n</code></pre> <p>Standard error of the value.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult","title":"ValuationResult","text":"<pre><code>ValuationResult(\n    *,\n    values: NDArray[float_],\n    variances: Optional[NDArray[float_]] = None,\n    counts: Optional[NDArray[int_]] = None,\n    indices: Optional[NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    algorithm: str = \"\",\n    status: Status = Status.Pending,\n    sort: bool = False,\n    **extra_values\n)\n</code></pre> <p>             Bases: <code>Sequence</code>, <code>Iterable[ValueItem[IndexT, NameT]]</code>, <code>Generic[IndexT, NameT]</code></p> <p>Objects of this class hold the results of valuation algorithms.</p> <p>These include indices in the original Dataset, any data names (e.g. group names in GroupedDataset), the values themselves, and variance of the computation in the case of Monte Carlo methods. <code>ValuationResults</code> can be iterated over like any <code>Sequence</code>: <code>iter(valuation_result)</code> returns a generator of ValueItem in the order in which the object is sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--indexing","title":"Indexing","text":"<p>Indexing can be position-based, when accessing any of the attributes values, variances, counts and indices, as well as when iterating over the object, or using the item access operator, both getter and setter. The \"position\" is either the original sequence in which the data was passed to the constructor, or the sequence in which the object is sorted, see below.</p> <p>Alternatively, indexing can be data-based, i.e. using the indices in the original dataset. This is the case for the methods get() and update().</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--sorting","title":"Sorting","text":"<p>Results can be sorted in-place with sort(), or alternatively using python's standard <code>sorted()</code> and <code>reversed()</code> Note that sorting values affects how iterators and the object itself as <code>Sequence</code> behave: <code>values[0]</code> returns a ValueItem with the highest or lowest ranking point if this object is sorted by descending or ascending value, respectively. If unsorted, <code>values[0]</code> returns the <code>ValueItem</code> at position 0, which has data index <code>indices[0]</code> in the Dataset.</p> <p>The same applies to direct indexing of the <code>ValuationResult</code>: the index is positional, according to the sorting. It does not refer to the \"data index\". To sort according to data index, use sort() with <code>key=\"index\"</code>.</p> <p>In order to access ValueItem objects by their data index, use get().</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--operating-on-results","title":"Operating on results","text":"<p>Results can be added to each other with the <code>+</code> operator. Means and variances are correctly updated, using the <code>counts</code> attribute.</p> <p>Results can also be updated with new values using update(). Means and variances are updated accordingly using the Welford algorithm.</p> <p>Empty objects behave in a special way, see empty().</p> PARAMETER  DESCRIPTION <code>values</code> <p>An array of values. If omitted, defaults to an empty array or to an array of zeros if <code>indices</code> are given.</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>indices</code> <p>An optional array of indices in the original dataset. If omitted, defaults to <code>np.arange(len(values))</code>. Warning: It is common to pass the indices of a Dataset here. Attention must be paid in a parallel context to copy them to the local process. Just do <code>indices=np.copy(data.indices)</code>.</p> <p> TYPE: <code>Optional[NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> <code>variances</code> <p>An optional array of variances in the computation of each value.</p> <p> TYPE: <code>Optional[NDArray[float_]]</code> DEFAULT: <code>None</code> </p> <code>counts</code> <p>An optional array with the number of updates for each value. Defaults to an array of ones.</p> <p> TYPE: <code>Optional[NDArray[int_]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>Names for the data points. Defaults to index numbers if not set.</p> <p> TYPE: <code>Optional[Sequence[NameT] | NDArray[NameT]]</code> DEFAULT: <code>None</code> </p> <code>algorithm</code> <p>The method used.</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> <code>status</code> <p>The end status of the algorithm.</p> <p> TYPE: <code>Status</code> DEFAULT: <code>Pending</code> </p> <code>sort</code> <p>Whether to sort the indices by ascending value. See above how this affects usage as an iterable or sequence.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>extra_values</code> <p>Additional values that can be passed as keyword arguments. This can contain, for example, the least core value.</p> <p> DEFAULT: <code>{}</code> </p> RAISES DESCRIPTION <code>ValueError</code> <p>If input arrays have mismatching lengths.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __init__(\n    self,\n    *,\n    values: NDArray[np.float_],\n    variances: Optional[NDArray[np.float_]] = None,\n    counts: Optional[NDArray[np.int_]] = None,\n    indices: Optional[NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    algorithm: str = \"\",\n    status: Status = Status.Pending,\n    sort: bool = False,\n    **extra_values,\n):\n    if variances is not None and len(variances) != len(values):\n        raise ValueError(\"Lengths of values and variances do not match\")\n    if data_names is not None and len(data_names) != len(values):\n        raise ValueError(\"Lengths of values and data_names do not match\")\n    if indices is not None and len(indices) != len(values):\n        raise ValueError(\"Lengths of values and indices do not match\")\n\n    self._algorithm = algorithm\n    self._status = Status(status)  # Just in case we are given a string\n    self._values = values\n    self._variances = np.zeros_like(values) if variances is None else variances\n    self._counts = np.ones_like(values) if counts is None else counts\n    self._sort_order = None\n    self._extra_values = extra_values or {}\n\n    # Yuk...\n    if data_names is None:\n        if indices is not None:\n            self._names = np.copy(indices)\n        else:\n            self._names = np.arange(len(self._values), dtype=np.int_)\n    elif not isinstance(data_names, np.ndarray):\n        self._names = np.array(data_names)\n    else:\n        self._names = data_names.copy()\n    if len(np.unique(self._names)) != len(self._names):\n        raise ValueError(\"Data names must be unique\")\n\n    if indices is None:\n        indices = np.arange(len(self._values), dtype=np.int_)\n    self._indices = indices\n    self._positions = {idx: pos for pos, idx in enumerate(indices)}\n\n    self._sort_positions: NDArray[np.int_] = np.arange(\n        len(self._values), dtype=np.int_\n    )\n    if sort:\n        self.sort()\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.values","title":"values  <code>property</code>","text":"<pre><code>values: NDArray[float_]\n</code></pre> <p>The values, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.variances","title":"variances  <code>property</code>","text":"<pre><code>variances: NDArray[float_]\n</code></pre> <p>The variances, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.stderr","title":"stderr  <code>property</code>","text":"<pre><code>stderr: NDArray[float_]\n</code></pre> <p>The raw standard errors, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.counts","title":"counts  <code>property</code>","text":"<pre><code>counts: NDArray[int_]\n</code></pre> <p>The raw counts, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.indices","title":"indices  <code>property</code>","text":"<pre><code>indices: NDArray[IndexT]\n</code></pre> <p>The indices for the values, possibly sorted.</p> <p>If the object is unsorted, then these are the same as declared at construction or <code>np.arange(len(values))</code> if none were passed.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.names","title":"names  <code>property</code>","text":"<pre><code>names: NDArray[NameT]\n</code></pre> <p>The names for the values, possibly sorted. If the object is unsorted, then these are the same as declared at construction or <code>np.arange(len(values))</code> if none were passed.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.sort","title":"sort","text":"<pre><code>sort(\n    reverse: bool = False,\n    key: Literal[\"value\", \"variance\", \"index\", \"name\"] = \"value\",\n) -&gt; None\n</code></pre> <p>Sorts the indices in place by <code>key</code>.</p> <p>Once sorted, iteration over the results, and indexing of all the properties ValuationResult.values, ValuationResult.variances, ValuationResult.counts, ValuationResult.indices and ValuationResult.names will follow the same order.</p> PARAMETER  DESCRIPTION <code>reverse</code> <p>Whether to sort in descending order by value.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>key</code> <p>The key to sort by. Defaults to ValueItem.value.</p> <p> TYPE: <code>Literal['value', 'variance', 'index', 'name']</code> DEFAULT: <code>'value'</code> </p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def sort(\n    self,\n    reverse: bool = False,\n    # Need a \"Comparable\" type here\n    key: Literal[\"value\", \"variance\", \"index\", \"name\"] = \"value\",\n) -&gt; None:\n    \"\"\"Sorts the indices in place by `key`.\n\n    Once sorted, iteration over the results, and indexing of all the\n    properties\n    [ValuationResult.values][pydvl.value.result.ValuationResult.values],\n    [ValuationResult.variances][pydvl.value.result.ValuationResult.variances],\n    [ValuationResult.counts][pydvl.value.result.ValuationResult.counts],\n    [ValuationResult.indices][pydvl.value.result.ValuationResult.indices]\n    and [ValuationResult.names][pydvl.value.result.ValuationResult.names]\n    will follow the same order.\n\n    Args:\n        reverse: Whether to sort in descending order by value.\n        key: The key to sort by. Defaults to\n            [ValueItem.value][pydvl.value.result.ValueItem].\n    \"\"\"\n    keymap = {\n        \"index\": \"_indices\",\n        \"value\": \"_values\",\n        \"variance\": \"_variances\",\n        \"name\": \"_names\",\n    }\n    self._sort_positions = np.argsort(getattr(self, keymap[key]))\n    if reverse:\n        self._sort_positions = self._sort_positions[::-1]\n    self._sort_order = reverse\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__getattr__","title":"__getattr__","text":"<pre><code>__getattr__(attr: str) -&gt; Any\n</code></pre> <p>Allows access to extra values as if they were properties of the instance.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __getattr__(self, attr: str) -&gt; Any:\n    \"\"\"Allows access to extra values as if they were properties of the instance.\"\"\"\n    # This is here to avoid a RecursionError when copying or pickling the object\n    if attr == \"_extra_values\":\n        raise AttributeError()\n    try:\n        return self._extra_values[attr]\n    except KeyError as e:\n        raise AttributeError(\n            f\"{self.__class__.__name__} object has no attribute {attr}\"\n        ) from e\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__iter__","title":"__iter__","text":"<pre><code>__iter__() -&gt; Iterator[ValueItem[IndexT, NameT]]\n</code></pre> <p>Iterate over the results returning ValueItem objects. To sort in place before iteration, use sort().</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __iter__(self) -&gt; Iterator[ValueItem[IndexT, NameT]]:\n    \"\"\"Iterate over the results returning [ValueItem][pydvl.value.result.ValueItem] objects.\n    To sort in place before iteration, use [sort()][pydvl.value.result.ValuationResult.sort].\n    \"\"\"\n    for pos in self._sort_positions:\n        yield ValueItem(\n            self._indices[pos],\n            self._names[pos],\n            self._values[pos],\n            self._variances[pos],\n            self._counts[pos],\n        )\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__add__","title":"__add__","text":"<pre><code>__add__(\n    other: ValuationResult[IndexT, NameT]\n) -&gt; ValuationResult[IndexT, NameT]\n</code></pre> <p>Adds two ValuationResults.</p> <p>The values must have been computed with the same algorithm. An exception to this is if one argument has empty values, in which case the other argument is returned.</p> <p>Warning</p> <p>Abusing this will introduce numerical errors.</p> <p>Means and standard errors are correctly handled. Statuses are added with bit-wise <code>&amp;</code>, see Status. <code>data_names</code> are taken from the left summand, or if unavailable from the right one. The <code>algorithm</code> string is carried over if both terms have the same one or concatenated.</p> <p>It is possible to add ValuationResults of different lengths, and with different or overlapping indices. The result will have the union of indices, and the values.</p> <p>Warning</p> <p>FIXME: Arbitrary <code>extra_values</code> aren't handled.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __add__(\n    self, other: ValuationResult[IndexT, NameT]\n) -&gt; ValuationResult[IndexT, NameT]:\n    \"\"\"Adds two ValuationResults.\n\n    The values must have been computed with the same algorithm. An exception\n    to this is if one argument has empty values, in which case the other\n    argument is returned.\n\n    !!! Warning\n        Abusing this will introduce numerical errors.\n\n    Means and standard errors are correctly handled. Statuses are added with\n    bit-wise `&amp;`, see [Status][pydvl.value.result.Status].\n    `data_names` are taken from the left summand, or if unavailable from\n    the right one. The `algorithm` string is carried over if both terms\n    have the same one or concatenated.\n\n    It is possible to add ValuationResults of different lengths, and with\n    different or overlapping indices. The result will have the union of\n    indices, and the values.\n\n    !!! Warning\n        FIXME: Arbitrary `extra_values` aren't handled.\n\n    \"\"\"\n    # empty results\n    if len(self.values) == 0:\n        return other\n    if len(other.values) == 0:\n        return self\n\n    self._check_compatible(other)\n\n    indices = np.union1d(self._indices, other._indices).astype(self._indices.dtype)\n    this_pos = np.searchsorted(indices, self._indices)\n    other_pos = np.searchsorted(indices, other._indices)\n\n    n: NDArray[np.int_] = np.zeros_like(indices, dtype=int)\n    m: NDArray[np.int_] = np.zeros_like(indices, dtype=int)\n    xn: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\n    xm: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\n    vn: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\n    vm: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\n\n    n[this_pos] = self._counts\n    xn[this_pos] = self._values\n    vn[this_pos] = self._variances\n    m[other_pos] = other._counts\n    xm[other_pos] = other._values\n    vm[other_pos] = other._variances\n\n    # np.maximum(1, n + m) covers case n = m = 0.\n    n_m_sum = np.maximum(1, n + m)\n\n    # Sample mean of n+m samples from two means of n and m samples\n    xnm = (n * xn + m * xm) / n_m_sum\n\n    # Sample variance of n+m samples from two sample variances of n and m samples\n    vnm = (n * (vn + xn**2) + m * (vm + xm**2)) / n_m_sum - xnm**2\n\n    if np.any(vnm &lt; 0):\n        if np.any(vnm &lt; -1e-6):\n            logger.warning(\n                \"Numerical error in variance computation. \"\n                f\"Negative sample variances clipped to 0 in {vnm}\"\n            )\n        vnm[np.where(vnm &lt; 0)] = 0\n\n    # Merging of names:\n    # If an index has the same name in both results, it must be the same.\n    # If an index has a name in one result but not the other, the name is\n    # taken from the result with the name.\n    if self._names.dtype != other._names.dtype:\n        if np.can_cast(other._names.dtype, self._names.dtype, casting=\"safe\"):\n            other._names = other._names.astype(self._names.dtype)\n            logger.warning(\n                f\"Casting ValuationResult.names from {other._names.dtype} to {self._names.dtype}\"\n            )\n        else:\n            raise TypeError(\n                f\"Cannot cast ValuationResult.names from \"\n                f\"{other._names.dtype} to {self._names.dtype}\"\n            )\n\n    both_pos = np.intersect1d(this_pos, other_pos)\n\n    if len(both_pos) &gt; 0:\n        this_names: NDArray = np.empty_like(indices, dtype=object)\n        other_names: NDArray = np.empty_like(indices, dtype=object)\n        this_names[this_pos] = self._names\n        other_names[other_pos] = other._names\n\n        this_shared_names = np.take(this_names, both_pos)\n        other_shared_names = np.take(other_names, both_pos)\n\n        if np.any(this_shared_names != other_shared_names):\n            raise ValueError(f\"Mismatching names in ValuationResults\")\n\n    names = np.empty_like(indices, dtype=self._names.dtype)\n    names[this_pos] = self._names\n    names[other_pos] = other._names\n\n    return ValuationResult(\n        algorithm=self.algorithm or other.algorithm or \"\",\n        status=self.status &amp; other.status,\n        indices=indices,\n        values=xnm,\n        variances=vnm,\n        counts=n + m,\n        data_names=names,\n        # FIXME: What to do with extra_values? This is not commutative:\n        # extra_values=self._extra_values.update(other._extra_values),\n    )\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.update","title":"update","text":"<pre><code>update(idx: int, new_value: float) -&gt; ValuationResult[IndexT, NameT]\n</code></pre> <p>Updates the result in place with a new value, using running mean and variance.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Data index of the value to update.</p> <p> TYPE: <code>int</code> </p> <code>new_value</code> <p>New value to add to the result.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>ValuationResult[IndexT, NameT]</code> <p>A reference to the same, modified result.</p> RAISES DESCRIPTION <code>IndexError</code> <p>If the index is not found.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def update(self, idx: int, new_value: float) -&gt; ValuationResult[IndexT, NameT]:\n    \"\"\"Updates the result in place with a new value, using running mean\n    and variance.\n\n    Args:\n        idx: Data index of the value to update.\n        new_value: New value to add to the result.\n\n    Returns:\n        A reference to the same, modified result.\n\n    Raises:\n        IndexError: If the index is not found.\n    \"\"\"\n    try:\n        pos = self._positions[idx]\n    except KeyError:\n        raise IndexError(f\"Index {idx} not found in ValuationResult\")\n    val, var = running_moments(\n        self._values[pos], self._variances[pos], self._counts[pos], new_value\n    )\n    self[pos] = ValueItem(\n        index=cast(IndexT, idx),  # FIXME\n        name=self._names[pos],\n        value=val,\n        variance=var,\n        count=self._counts[pos] + 1,\n    )\n    return self\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.scale","title":"scale","text":"<pre><code>scale(factor: float, indices: Optional[NDArray[IndexT]] = None)\n</code></pre> <p>Scales the values and variances of the result by a coefficient.</p> PARAMETER  DESCRIPTION <code>factor</code> <p>Factor to scale by.</p> <p> TYPE: <code>float</code> </p> <code>indices</code> <p>Indices to scale. If None, all values are scaled.</p> <p> TYPE: <code>Optional[NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def scale(self, factor: float, indices: Optional[NDArray[IndexT]] = None):\n    \"\"\"\n    Scales the values and variances of the result by a coefficient.\n\n    Args:\n        factor: Factor to scale by.\n        indices: Indices to scale. If None, all values are scaled.\n    \"\"\"\n    self._values[self._sort_positions[indices]] *= factor\n    self._variances[self._sort_positions[indices]] *= factor**2\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.get","title":"get","text":"<pre><code>get(idx: Integral) -&gt; ValueItem\n</code></pre> <p>Retrieves a ValueItem by data index, as opposed to sort index, like the indexing operator.</p> RAISES DESCRIPTION <code>IndexError</code> <p>If the index is not found.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def get(self, idx: Integral) -&gt; ValueItem:\n    \"\"\"Retrieves a ValueItem by data index, as opposed to sort index, like\n    the indexing operator.\n\n    Raises:\n         IndexError: If the index is not found.\n    \"\"\"\n    try:\n        pos = self._positions[idx]\n    except KeyError:\n        raise IndexError(f\"Index {idx} not found in ValuationResult\")\n\n    return ValueItem(\n        self._indices[pos],\n        self._names[pos],\n        self._values[pos],\n        self._variances[pos],\n        self._counts[pos],\n    )\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.to_dataframe","title":"to_dataframe","text":"<pre><code>to_dataframe(\n    column: Optional[str] = None, use_names: bool = False\n) -&gt; DataFrame\n</code></pre> <p>Returns values as a dataframe.</p> PARAMETER  DESCRIPTION <code>column</code> <p>Name for the column holding the data value. Defaults to the name of the algorithm used.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>use_names</code> <p>Whether to use data names instead of indices for the DataFrame's index.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>DataFrame</code> <p>A dataframe with two columns, one for the values, with name given as explained in <code>column</code>, and another with standard errors for approximate algorithms. The latter will be named <code>column+'_stderr'</code>.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def to_dataframe(\n    self, column: Optional[str] = None, use_names: bool = False\n) -&gt; pd.DataFrame:\n    \"\"\"Returns values as a dataframe.\n\n    Args:\n        column: Name for the column holding the data value. Defaults to\n            the name of the algorithm used.\n        use_names: Whether to use data names instead of indices for the\n            DataFrame's index.\n\n    Returns:\n        A dataframe with two columns, one for the values, with name\n            given as explained in `column`, and another with standard errors for\n            approximate algorithms. The latter will be named `column+'_stderr'`.\n    \"\"\"\n    column = column or self._algorithm\n    df = pd.DataFrame(\n        self._values[self._sort_positions],\n        index=(\n            self._names[self._sort_positions]\n            if use_names\n            else self._indices[self._sort_positions]\n        ),\n        columns=[column],\n    )\n    df[column + \"_stderr\"] = self.stderr[self._sort_positions]\n    df[column + \"_updates\"] = self.counts[self._sort_positions]\n    return df\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.from_random","title":"from_random  <code>classmethod</code>","text":"<pre><code>from_random(\n    size: int,\n    total: Optional[float] = None,\n    seed: Optional[Seed] = None,\n    **kwargs\n) -&gt; \"ValuationResult\"\n</code></pre> <p>Creates a ValuationResult object and fills it with an array of random values from a uniform distribution in [-1,1]. The values can be made to sum up to a given total number (doing so will change their range).</p> PARAMETER  DESCRIPTION <code>size</code> <p>Number of values to generate</p> <p> TYPE: <code>int</code> </p> <code>total</code> <p>If set, the values are normalized to sum to this number (\"efficiency\" property of Shapley values).</p> <p> TYPE: <code>Optional[float]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional options to pass to the constructor of ValuationResult. Use to override status, names, etc.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>'ValuationResult'</code> <p>A valuation result with its status set to</p> <code>'ValuationResult'</code> <p>Status.Converged by default.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>size</code> is less than 1.</p> <p>Changed in version 0.6.0</p> <p>Added parameter <code>total</code>. Check for zero size</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\ndef from_random(\n    cls,\n    size: int,\n    total: Optional[float] = None,\n    seed: Optional[Seed] = None,\n    **kwargs,\n) -&gt; \"ValuationResult\":\n    \"\"\"Creates a [ValuationResult][pydvl.value.result.ValuationResult] object and fills it with an array\n    of random values from a uniform distribution in [-1,1]. The values can\n    be made to sum up to a given total number (doing so will change their range).\n\n    Args:\n        size: Number of values to generate\n        total: If set, the values are normalized to sum to this number\n            (\"efficiency\" property of Shapley values).\n        kwargs: Additional options to pass to the constructor of\n            [ValuationResult][pydvl.value.result.ValuationResult]. Use to override status, names, etc.\n\n    Returns:\n        A valuation result with its status set to\n        [Status.Converged][pydvl.utils.status.Status] by default.\n\n    Raises:\n         ValueError: If `size` is less than 1.\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added parameter `total`. Check for zero size\n    \"\"\"\n    if size &lt; 1:\n        raise ValueError(\"Size must be a positive integer\")\n\n    rng = np.random.default_rng(seed)\n    values = rng.uniform(low=-1, high=1, size=size)\n    if total is not None:\n        values *= total / np.sum(values)\n\n    options = dict(values=values, status=Status.Converged, algorithm=\"random\")\n    options.update(kwargs)\n    return cls(**options)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.empty","title":"empty  <code>classmethod</code>","text":"<pre><code>empty(\n    algorithm: str = \"\",\n    indices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    n_samples: int = 0,\n) -&gt; ValuationResult\n</code></pre> <p>Creates an empty ValuationResult object.</p> <p>Empty results are characterised by having an empty array of values. When another result is added to an empty one, the empty one is discarded.</p> PARAMETER  DESCRIPTION <code>algorithm</code> <p>Name of the algorithm used to compute the values</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> <code>indices</code> <p>Optional sequence or array of indices.</p> <p> TYPE: <code>Optional[Sequence[IndexT] | NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>Optional sequences or array of names for the data points. Defaults to index numbers if not set.</p> <p> TYPE: <code>Optional[Sequence[NameT] | NDArray[NameT]]</code> DEFAULT: <code>None</code> </p> <code>n_samples</code> <p>Number of valuation result entries.</p> <p> TYPE: <code>int</code> DEFAULT: <code>0</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\ndef empty(\n    cls,\n    algorithm: str = \"\",\n    indices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    n_samples: int = 0,\n) -&gt; ValuationResult:\n    \"\"\"Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.\n\n    Empty results are characterised by having an empty array of values. When\n    another result is added to an empty one, the empty one is discarded.\n\n    Args:\n        algorithm: Name of the algorithm used to compute the values\n        indices: Optional sequence or array of indices.\n        data_names: Optional sequences or array of names for the data points.\n            Defaults to index numbers if not set.\n        n_samples: Number of valuation result entries.\n\n    Returns:\n        Object with the results.\n    \"\"\"\n    if indices is not None or data_names is not None or n_samples != 0:\n        return cls.zeros(\n            algorithm=algorithm,\n            indices=indices,\n            data_names=data_names,\n            n_samples=n_samples,\n        )\n    return cls(algorithm=algorithm, status=Status.Pending, values=np.array([]))\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.zeros","title":"zeros  <code>classmethod</code>","text":"<pre><code>zeros(\n    algorithm: str = \"\",\n    indices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    n_samples: int = 0,\n) -&gt; ValuationResult\n</code></pre> <p>Creates an empty ValuationResult object.</p> <p>Empty results are characterised by having an empty array of values. When another result is added to an empty one, the empty one is ignored.</p> PARAMETER  DESCRIPTION <code>algorithm</code> <p>Name of the algorithm used to compute the values</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> <code>indices</code> <p>Data indices to use. A copy will be made. If not given, the indices will be set to the range <code>[0, n_samples)</code>.</p> <p> TYPE: <code>Optional[Sequence[IndexT] | NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>Data names to use. A copy will be made. If not given, the names will be set to the string representation of the indices.</p> <p> TYPE: <code>Optional[Sequence[NameT] | NDArray[NameT]]</code> DEFAULT: <code>None</code> </p> <code>n_samples</code> <p>Number of data points whose values are computed. If not given, the length of <code>indices</code> will be used.</p> <p> TYPE: <code>int</code> DEFAULT: <code>0</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\ndef zeros(\n    cls,\n    algorithm: str = \"\",\n    indices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    n_samples: int = 0,\n) -&gt; ValuationResult:\n    \"\"\"Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.\n\n    Empty results are characterised by having an empty array of values. When\n    another result is added to an empty one, the empty one is ignored.\n\n    Args:\n        algorithm: Name of the algorithm used to compute the values\n        indices: Data indices to use. A copy will be made. If not given,\n            the indices will be set to the range `[0, n_samples)`.\n        data_names: Data names to use. A copy will be made. If not given,\n            the names will be set to the string representation of the indices.\n        n_samples: Number of data points whose values are computed. If\n            not given, the length of `indices` will be used.\n\n    Returns:\n        Object with the results.\n    \"\"\"\n    if indices is None:\n        indices = np.arange(n_samples, dtype=np.int_)\n    else:\n        indices = np.array(indices, dtype=np.int_)\n\n    if data_names is None:\n        data_names = np.array(indices)\n    else:\n        data_names = np.array(data_names)\n\n    return cls(\n        algorithm=algorithm,\n        status=Status.Pending,\n        indices=indices,\n        data_names=data_names,\n        values=np.zeros(len(indices)),\n        variances=np.zeros(len(indices)),\n        counts=np.zeros(len(indices), dtype=np.int_),\n    )\n</code></pre>"},{"location":"api/pydvl/value/sampler/","title":"Sampler","text":""},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler","title":"pydvl.value.sampler","text":"<p>Samplers iterate over subsets of indices.</p> <p>The classes in this module are used to iterate over indices and subsets of their complement in the whole set, as required for the computation of marginal utility for semi-values. The elements returned when iterating over any subclass of PowersetSampler are tuples of the form <code>(idx, subset)</code>, where <code>idx</code> is the index of the element being added to the subset, and <code>subset</code> is the subset of the complement of <code>idx</code>. The classes in this module are used to iterate over an index set \\(I\\) as required for the computation of marginal utility for semi-values. The elements returned when iterating over any subclass of :class:<code>PowersetSampler</code> are tuples of the form \\((i, S)\\), where \\(i\\) is an index of interest, and \\(S \\subset I \\setminus \\{i\\}\\) is a subset of the complement of \\(i\\).</p> <p>The iteration happens in two nested loops. An outer loop iterates over \\(I\\), and an inner loop iterates over the powerset of \\(I \\setminus \\{i\\}\\). The outer iteration can be either sequential or at random.</p> <p>Note</p> <p>This is the natural mode of iteration for the combinatorial definition of semi-values, in particular Shapley value. For the computation using permutations, adhering to this interface is not ideal, but we stick to it for consistency.</p> <p>The samplers are used in the semivalues module to compute any semi-value, in particular Shapley and Beta values, and Banzhaf indices.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler--slicing-of-samplers","title":"Slicing of samplers","text":"<p>The samplers can be sliced for parallel computation. For those which are embarrassingly parallel, this is done by slicing the set of \"outer\" indices and returning new samplers over those slices. This includes all truly powerset-based samplers, such as DeterministicUniformSampler and UniformSampler. In contrast, slicing a PermutationSampler creates a new sampler which iterates over the same indices.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler--references","title":"References","text":"<ol> <li> <p>Mitchell, Rory, Joshua Cooper, Eibe   Frank, and Geoffrey Holmes. Sampling Permutations for Shapley Value   Estimation. Journal of Machine   Learning Research 23, no. 43 (2022): 1\u201346.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T. and Jia, R., 2023. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning. In: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics, pp. 6388-6421.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler","title":"PowersetSampler","text":"<pre><code>PowersetSampler(\n    indices: NDArray[IndexT],\n    index_iteration: IndexIteration = IndexIteration.Sequential,\n    outer_indices: NDArray[IndexT] | None = None,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>ABC</code>, <code>Iterable[SampleT]</code>, <code>Generic[IndexT]</code></p> <p>Samplers are custom iterables over subsets of indices.</p> <p>Calling <code>iter()</code> on a sampler returns an iterator over tuples of the form \\((i, S)\\), where \\(i\\) is an index of interest, and \\(S \\subset I \\setminus \\{i\\}\\) is a subset of the complement of \\(i\\).</p> <p>This is done in two nested loops, where the outer loop iterates over the set of indices, and the inner loop iterates over subsets of the complement of the current index. The outer iteration can be either sequential or at random.</p> <p>Note</p> <p>Samplers are not iterators themselves, so that each call to <code>iter()</code> e.g. in a for loop creates a new iterator.</p> Example <pre><code>&gt;&gt;&gt;for idx, s in DeterministicUniformSampler(np.arange(2)):\n&gt;&gt;&gt;    print(s, end=\"\")\n[][2,][][1,]\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler--methods-required-in-subclasses","title":"Methods required in subclasses","text":"<p>Samplers must implement a weight() function to be used as a multiplier in Monte Carlo sums, so that the limit expectation coincides with the semi-value.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler--slicing-of-samplers","title":"Slicing of samplers","text":"<p>The samplers can be sliced for parallel computation. For those which are embarrassingly parallel, this is done by slicing the set of \"outer\" indices and returning new samplers over those slices.</p> <pre><code>index_iteration: the order in which indices are iterated over\nouter_indices: The set of items (indices) over which to iterate\n    when sampling. Subsets are taken from the complement of each index\n    in succession. For embarrassingly parallel computations, this set\n    is sliced and the samplers are used to iterate over the slices.\n</code></pre> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(\n    self,\n    indices: NDArray[IndexT],\n    index_iteration: IndexIteration = IndexIteration.Sequential,\n    outer_indices: NDArray[IndexT] | None = None,\n    **kwargs,\n):\n    \"\"\"\n    Args:\n        indices: The set of items (indices) to sample from.\n        index_iteration: the order in which indices are iterated over\n        outer_indices: The set of items (indices) over which to iterate\n            when sampling. Subsets are taken from the complement of each index\n            in succession. For embarrassingly parallel computations, this set\n            is sliced and the samplers are used to iterate over the slices.\n    \"\"\"\n    self._indices = indices\n    self._index_iteration = index_iteration\n    self._outer_indices = outer_indices if outer_indices is not None else indices\n    self._n = len(indices)\n    self._n_samples = 0\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.weight","title":"weight  <code>abstractmethod</code> <code>classmethod</code>","text":"<pre><code>weight(n: int, subset_len: int) -&gt; float\n</code></pre> <p>Factor by which to multiply Monte Carlo samples, so that the mean converges to the desired expression.</p> <p>By the Law of Large Numbers, the sample mean of \\(\\delta_i(S_j)\\) converges to the expectation under the distribution from which \\(S_j\\) is sampled.</p> \\[ \\frac{1}{m}  \\sum_{j = 1}^m \\delta_i (S_j) c (S_j) \\longrightarrow    \\underset{S \\sim \\mathcal{D}_{- i}}{\\mathbb{E}} [\\delta_i (S) c (    S)]\\] <p>We add a factor \\(c(S_j)\\) in order to have this expectation coincide with the desired expression.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>@classmethod\n@abc.abstractmethod\ndef weight(cls, n: int, subset_len: int) -&gt; float:\n    r\"\"\"Factor by which to multiply Monte Carlo samples, so that the\n    mean converges to the desired expression.\n\n    By the Law of Large Numbers, the sample mean of $\\delta_i(S_j)$\n    converges to the expectation under the distribution from which $S_j$ is\n    sampled.\n\n    $$ \\frac{1}{m}  \\sum_{j = 1}^m \\delta_i (S_j) c (S_j) \\longrightarrow\n       \\underset{S \\sim \\mathcal{D}_{- i}}{\\mathbb{E}} [\\delta_i (S) c (\n       S)]$$\n\n    We add a factor $c(S_j)$ in order to have this expectation coincide with\n    the desired expression.\n    \"\"\"\n    ...\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.StochasticSamplerMixin","title":"StochasticSamplerMixin","text":"<pre><code>StochasticSamplerMixin(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>Mixin class for samplers which use a random number generator.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicUniformSampler","title":"DeterministicUniformSampler","text":"<pre><code>DeterministicUniformSampler(indices: NDArray[IndexT], *args, **kwargs)\n</code></pre> <p>             Bases: <code>PowersetSampler[IndexT]</code></p> <p>For every index \\(i\\), each subset of the complement <code>indices - {i}</code> is returned.</p> <p>Note</p> <p>Indices are always iterated over sequentially, irrespective of the value of <code>index_iteration</code> upon construction.</p> Example <pre><code>&gt;&gt;&gt; for idx, s in DeterministicUniformSampler(np.arange(2)):\n&gt;&gt;&gt;    print(f\"{idx} - {s}\", end=\", \")\n1 - [], 1 - [2], 2 - [], 2 - [1],\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>The set of items (indices) to sample from.</p> <p> TYPE: <code>NDArray[IndexT]</code> </p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, indices: NDArray[IndexT], *args, **kwargs):\n    \"\"\"An iterator to perform uniform deterministic sampling of subsets.\n\n    For every index $i$, each subset of the complement `indices - {i}` is\n    returned.\n\n    !!! Note\n        Indices are always iterated over sequentially, irrespective of\n        the value of `index_iteration` upon construction.\n\n    ??? Example\n        ``` pycon\n        &gt;&gt;&gt; for idx, s in DeterministicUniformSampler(np.arange(2)):\n        &gt;&gt;&gt;    print(f\"{idx} - {s}\", end=\", \")\n        1 - [], 1 - [2], 2 - [], 2 - [1],\n        ```\n\n    Args:\n        indices: The set of items (indices) to sample from.\n    \"\"\"\n    # Force sequential iteration\n    kwargs.update({\"index_iteration\": PowersetSampler.IndexIteration.Sequential})\n    super().__init__(indices, *args, **kwargs)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicUniformSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicUniformSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler","title":"UniformSampler","text":"<pre><code>UniformSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>An iterator to perform uniform random sampling of subsets.</p> <p>Iterating over every index \\(i\\), either in sequence or at random depending on the value of <code>index_iteration</code>, one subset of the complement <code>indices - {i}</code> is sampled with equal probability \\(2^{n-1}\\). The iterator never ends.</p> Example <p>The code <pre><code>for idx, s in UniformSampler(np.arange(3)):\n   print(f\"{idx} - {s}\", end=\", \")\n</code></pre> Produces the output: <pre><code>0 - [1 4], 1 - [2 3], 2 - [0 1 3], 3 - [], 4 - [2], 0 - [1 3 4], 1 - [0 2]\n(...)\n</code></pre></p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler.weight","title":"weight  <code>classmethod</code>","text":"<pre><code>weight(n: int, subset_len: int) -&gt; float\n</code></pre> <p>Correction coming from Monte Carlo integration so that the mean of the marginals converges to the value: the uniform distribution over the powerset of a set with n-1 elements has mass 2^{n-1} over each subset.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>@classmethod\ndef weight(cls, n: int, subset_len: int) -&gt; float:\n    \"\"\"Correction coming from Monte Carlo integration so that the mean of\n    the marginals converges to the value: the uniform distribution over the\n    powerset of a set with n-1 elements has mass 2^{n-1} over each subset.\"\"\"\n    return float(2 ** (n - 1)) if n &gt; 0 else 1.0\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.MSRSampler","title":"MSRSampler","text":"<pre><code>MSRSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>An iterator to perform sampling of random subsets.</p> <p>This sampler does not return any index, it only returns subsets of the data. This sampler is used in (Wang et. al.)<sup>2</sup>.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.MSRSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.MSRSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticSampler","title":"AntitheticSampler","text":"<pre><code>AntitheticSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>An iterator to perform uniform random sampling of subsets, and their complements.</p> <p>Works as UniformSampler, but for every tuple \\((i,S)\\), it subsequently returns \\((i,S^c)\\), where \\(S^c\\) is the complement of the set \\(S\\) in the set of indices, excluding \\(i\\).</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler","title":"PermutationSampler","text":"<pre><code>PermutationSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>Sample permutations of indices and iterate through each returning increasing subsets, as required for the permutation definition of semi-values.</p> <p>This sampler does not implement the two loops described in PowersetSampler. Instead, for a permutation <code>(3,1,4,2)</code>, it returns in sequence the tuples of index and sets:  <code>(3, {})</code>, <code>(1, {3})</code>, <code>(4, {3,1})</code> and <code>(2, {3,1,4})</code>.</p> <p>Note that the full index set is never returned.</p> <p>Warning</p> <p>This sampler requires caching to be enabled or computation will be doubled wrt. a \"direct\" implementation of permutation MC</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler.__getitem__","title":"__getitem__","text":"<pre><code>__getitem__(key: slice | list[int]) -&gt; PowersetSampler[IndexT]\n</code></pre> <p>Permutation samplers cannot be split across indices, so we return a copy of the full sampler.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __getitem__(self, key: slice | list[int]) -&gt; PowersetSampler[IndexT]:\n    \"\"\"Permutation samplers cannot be split across indices, so we return\n    a copy of the full sampler.\"\"\"\n    return super().__getitem__(slice(None))\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticPermutationSampler","title":"AntitheticPermutationSampler","text":"<pre><code>AntitheticPermutationSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>PermutationSampler[IndexT]</code></p> <p>Samples permutations like PermutationSampler, but after each permutation, it returns the same permutation in reverse order.</p> <p>This sampler was suggested in (Mitchell et al. 2022)<sup>1</sup></p> <p>New in version 0.7.1</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticPermutationSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticPermutationSampler.__getitem__","title":"__getitem__","text":"<pre><code>__getitem__(key: slice | list[int]) -&gt; PowersetSampler[IndexT]\n</code></pre> <p>Permutation samplers cannot be split across indices, so we return a copy of the full sampler.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __getitem__(self, key: slice | list[int]) -&gt; PowersetSampler[IndexT]:\n    \"\"\"Permutation samplers cannot be split across indices, so we return\n    a copy of the full sampler.\"\"\"\n    return super().__getitem__(slice(None))\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticPermutationSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicPermutationSampler","title":"DeterministicPermutationSampler","text":"<pre><code>DeterministicPermutationSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>PermutationSampler[IndexT]</code></p> <p>Samples all n! permutations of the indices deterministically, and iterates through them, returning sets as required for the permutation-based definition of semi-values.</p> <p>Warning</p> <p>This sampler requires caching to be enabled or computation will be doubled wrt. a \"direct\" implementation of permutation MC</p> <p>Warning</p> <p>This sampler is not parallelizable, as it always iterates over the whole set of permutations in the same order. Different processes would always return the same values for all indices.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicPermutationSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicPermutationSampler.__getitem__","title":"__getitem__","text":"<pre><code>__getitem__(key: slice | list[int]) -&gt; PowersetSampler[IndexT]\n</code></pre> <p>Permutation samplers cannot be split across indices, so we return a copy of the full sampler.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __getitem__(self, key: slice | list[int]) -&gt; PowersetSampler[IndexT]:\n    \"\"\"Permutation samplers cannot be split across indices, so we return\n    a copy of the full sampler.\"\"\"\n    return super().__getitem__(slice(None))\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicPermutationSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.RandomHierarchicalSampler","title":"RandomHierarchicalSampler","text":"<pre><code>RandomHierarchicalSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>For every index, sample a set size, then a set of that size.</p> <p>Todo</p> <p>This is unnecessary, but a step towards proper stratified sampling.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.RandomHierarchicalSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.RandomHierarchicalSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/","title":"Semivalues","text":""},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues","title":"pydvl.value.semivalues","text":"<p>This module provides the core functionality for the computation of generic semi-values. A semi-value is any valuation function with the form:</p> \\[v_\\text{semi}(i) = \\sum_{i=1}^n w(k) \\sum_{S \\subset D_{-i}^{(k)}} [U(S_{+i})-U(S)],\\] <p>where the coefficients \\(w(k)\\) satisfy the property:</p> \\[\\sum_{k=1}^n w(k) = 1.\\] Note <p>For implementation consistency, we slightly depart from the common definition of semi-values, which includes a factor \\(1/n\\) in the sum over subsets. Instead, we subsume this factor into the coefficient \\(w(k)\\).</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues--main-components","title":"Main components","text":"<p>The computation of a semi-value requires two components:</p> <ol> <li>A subset sampler that generates subsets of the set \\(D\\) of interest.</li> <li>A coefficient \\(w(k)\\) that assigns a weight to each subset size \\(k\\).</li> </ol> <p>Samplers can be found in sampler, and can be classified into two categories: powerset samplers and permutation samplers. Powerset samplers generate subsets of \\(D_{-i}\\), while the permutation sampler generates permutations of \\(D\\). The former conform to the above definition of semi-values, while the latter reformulates it as:</p> \\[ v(i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} \\tilde{w}( | \\sigma_{:i} | )[U(\\sigma_{:i} \\cup \\{i\\}) \u2212 U(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears (see Data valuation for details), and</p> \\[ \\tilde{w} (k) = n \\binom{n - 1}{k} w (k) \\] <p>is the weight correction due to the reformulation.</p> <p>Warning</p> <p>Both PermutationSampler and DeterministicPermutationSampler require caching to be enabled or computation will be doubled wrt. a 'direct' implementation of permutation MC.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues--computing-semi-values","title":"Computing semi-values","text":"<p>Samplers and coefficients can be arbitrarily mixed by means of the main entry point of this module, compute_generic_semivalues. There are several pre-defined coefficients, including the Shapley value of (Ghorbani and Zou, 2019)<sup>1</sup>, the Banzhaf index of (Wang and Jia)<sup>3</sup>, and the Beta coefficient of (Kwon and Zou, 2022)<sup>2</sup>. For each of these methods, there is a convenience wrapper function. Respectively, these are: compute_shapley_semivalues, compute_banzhaf_semivalues, and compute_beta_shapley_semivalues. instead.</p> <p>Parallelization and batching</p> <p>In order to ensure reproducibility and fine-grained control of parallelization, samples are generated in the main process and then distributed to worker processes for evaluation. For small sample sizes, this can lead to a significant overhead. To avoid this, we temporarily provide an additional argument <code>batch_size</code> to all methods which can improve performance with small models up to an order of magnitude. Note that this argument will be removed before version 1.0 in favour of a more general solution.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y. and Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning. In: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022, Vol. 151. PMLR, Valencia, Spain.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T. and Jia, R., 2023. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning. In: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics, pp. 6388-6421.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SVCoefficient","title":"SVCoefficient","text":"<p>             Bases: <code>Protocol</code></p> <p>The protocol that coefficients for the computation of semi-values must fulfill.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SVCoefficient.__call__","title":"__call__","text":"<pre><code>__call__(n: int, k: int) -&gt; float\n</code></pre> <p>Computes the coefficient for a given subset size.</p> PARAMETER  DESCRIPTION <code>n</code> <p>Total number of elements in the set.</p> <p> TYPE: <code>int</code> </p> <code>k</code> <p>Size of the subset for which the coefficient is being computed</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __call__(self, n: int, k: int) -&gt; float:\n    \"\"\"Computes the coefficient for a given subset size.\n\n    Args:\n        n: Total number of elements in the set.\n        k: Size of the subset for which the coefficient is being computed\n    \"\"\"\n    ...\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.DefaultMarginal","title":"DefaultMarginal","text":"<p>             Bases: <code>MarginalFunction</code></p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.DefaultMarginal.__call__","title":"__call__","text":"<pre><code>__call__(\n    u: Utility, coefficient: SVCoefficient, samples: Iterable[SampleT]\n) -&gt; Tuple[MarginalT, ...]\n</code></pre> <p>Computation of marginal utility. This is a helper function for compute_generic_semivalues.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>coefficient</code> <p>The semivalue coefficient and sampler weight</p> <p> TYPE: <code>SVCoefficient</code> </p> <code>samples</code> <p>A collection of samples. Each sample is a tuple of index and subset of indices to compute a marginal utility.</p> <p> TYPE: <code>Iterable[SampleT]</code> </p> RETURNS DESCRIPTION <code>MarginalT</code> <p>A collection of marginals. Each marginal is a tuple with index and its marginal</p> <code>...</code> <p>utility.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __call__(\n    self, u: Utility, coefficient: SVCoefficient, samples: Iterable[SampleT]\n) -&gt; Tuple[MarginalT, ...]:\n    \"\"\"Computation of marginal utility. This is a helper function for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        coefficient: The semivalue coefficient and sampler weight\n        samples: A collection of samples. Each sample is a tuple of index and subset of\n            indices to compute a marginal utility.\n\n    Returns:\n        A collection of marginals. Each marginal is a tuple with index and its marginal\n        utility.\n    \"\"\"\n    n = len(u.data)\n    marginals: List[MarginalT] = []\n    for idx, s in samples:\n        marginal = (u({idx}.union(s)) - u(s)) * coefficient(n, len(s))\n        marginals.append((idx, marginal))\n    return tuple(marginals)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.RawUtility","title":"RawUtility","text":"<p>             Bases: <code>MarginalFunction</code></p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.RawUtility.__call__","title":"__call__","text":"<pre><code>__call__(\n    u: Utility, coefficient: SVCoefficient, samples: Iterable[SampleT]\n) -&gt; Tuple[MarginalT, ...]\n</code></pre> <p>Computation of raw utility without marginalization. This is a helper function for compute_generic_semivalues.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>coefficient</code> <p>The semivalue coefficient and sampler weight</p> <p> TYPE: <code>SVCoefficient</code> </p> <code>samples</code> <p>A collection of samples. Each sample is a tuple of index and subset of indices to compute a marginal utility.</p> <p> TYPE: <code>Iterable[SampleT]</code> </p> RETURNS DESCRIPTION <code>Tuple[MarginalT, ...]</code> <p>A collection of marginals. Each marginal is a tuple with index and its raw utility.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __call__(\n    self, u: Utility, coefficient: SVCoefficient, samples: Iterable[SampleT]\n) -&gt; Tuple[MarginalT, ...]:\n    \"\"\"Computation of raw utility without marginalization. This is a helper function for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        coefficient: The semivalue coefficient and sampler weight\n        samples: A collection of samples. Each sample is a tuple of index and subset of\n            indices to compute a marginal utility.\n\n    Returns:\n        A collection of marginals. Each marginal is a tuple with index and its raw utility.\n    \"\"\"\n    marginals: List[MarginalT] = []\n    for idx, s in samples:\n        marginals.append((s, u(s)))\n    return tuple(marginals)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.FutureProcessor","title":"FutureProcessor","text":"<p>The FutureProcessor class used to process the results of the parallel marginal evaluations.</p> <p>The marginals are evaluated in parallel by <code>n_jobs</code> threads, but some algorithms require a central method to postprocess the marginal results. This can be achieved through the future processor. This base class does not perform any postprocessing, it is a noop used in most data valuation algorithms.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.MSRFutureProcessor","title":"MSRFutureProcessor","text":"<pre><code>MSRFutureProcessor(u: Utility)\n</code></pre> <p>             Bases: <code>FutureProcessor</code></p> <p>This FutureProcessor processes the raw marginals in a way that MSR sampling requires.</p> <p>MSR sampling evaluates the utility once, and then updates all data semivalues based on this one evaluation. In order to do this, the RawUtility value needs to be postprocessed through this class. For more details on MSR, please refer to the paper (Wang et. al.)<sup>3</sup>. This processor keeps track of the current values and computes marginals for all data points, so that the values in the ValuationResult can be updated properly down the line.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __init__(self, u: Utility):\n    self.n = len(u.data)\n    self.all_indices = u.data.indices.copy()\n    self.point_in_subset = np.zeros((self.n,))\n    self.positive_sums = np.zeros((self.n,))\n    self.negative_sums = np.zeros((self.n,))\n    self.total_evaluations = 0\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.MSRFutureProcessor.__call__","title":"__call__","text":"<pre><code>__call__(\n    future_result: List[Tuple[List[IndexT], float]]\n) -&gt; List[List[MarginalT]]\n</code></pre> <p>Computation of marginal utility using Maximum Sample Reuse.</p> <pre><code>    This processor requires the Marginal Function to be set to RawUtility.\n    Then, this processor computes marginals based on the utility value and the index set provided.\n\n    The final formula that gives the Banzhaf semivalue using MSR is:\n    $$\\hat{\\phi}_{MSR}(i) = \frac{1}{|\\mathbf{S}_{\n</code></pre> <p>i i}|} \\sum_{S \\in \\mathbf{S}{ i i}} U(S)         - \frac{1}{|\\mathbf{S}{ ot{ i} i}|} \\sum_{S \\in \\mathbf{S}_{ ot{ i} i}} U(S)$$</p> <pre><code>    Args:\n        future_result: Result of the parallel computing jobs comprised of\n            a list of indices that were used to evaluate the utility, and the evaluation result (metric).\n\n    Returns:\n        A collection of marginals. Each marginal is a tuple with index and its marginal\n        utility.\n</code></pre> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __call__(\n    self, future_result: List[Tuple[List[IndexT], float]]\n) -&gt; List[List[MarginalT]]:\n    \"\"\"Computation of marginal utility using Maximum Sample Reuse.\n\n    This processor requires the Marginal Function to be set to RawUtility.\n    Then, this processor computes marginals based on the utility value and the index set provided.\n\n    The final formula that gives the Banzhaf semivalue using MSR is:\n    $$\\hat{\\phi}_{MSR}(i) = \\frac{1}{|\\mathbf{S}_{\\ni i}|} \\sum_{S \\in \\mathbf{S}_{\\ni i}} U(S)\n    - \\frac{1}{|\\mathbf{S}_{\\not{\\ni} i}|} \\sum_{S \\in \\mathbf{S}_{\\not{\\ni} i}} U(S)$$\n\n    Args:\n        future_result: Result of the parallel computing jobs comprised of\n            a list of indices that were used to evaluate the utility, and the evaluation result (metric).\n\n    Returns:\n        A collection of marginals. Each marginal is a tuple with index and its marginal\n        utility.\n    \"\"\"\n    marginals: List[List[MarginalT]] = []\n    for batch_id, (s, evaluation) in enumerate(future_result):\n        previous_values = self.compute_values()\n        self.total_evaluations += 1\n        self.point_in_subset[s] += 1\n        self.positive_sums[s] += evaluation\n        not_s = np.setdiff1d(self.all_indices, s)\n        self.negative_sums[not_s] += evaluation\n        new_values = self.compute_values()\n        # Hack to work around the update mechanic that does not work out of the box for MSR\n        marginal_vals = (\n            self.total_evaluations * new_values\n            - (self.total_evaluations - 1) * previous_values\n        )\n        marginals.append([])\n        for data_index in range(self.n):\n            marginals[batch_id].append(\n                (data_index, float(marginal_vals[data_index]))\n            )\n    return marginals\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SemiValueMode","title":"SemiValueMode","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Enumeration of semi-value modes.</p> <p>Deprecation notice</p> <p>This enum and the associated methods are deprecated and will be removed in 0.8.0.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_generic_semivalues","title":"compute_generic_semivalues","text":"<pre><code>compute_generic_semivalues(\n    sampler: PowersetSampler[IndexT],\n    u: Utility,\n    coefficient: SVCoefficient,\n    done: StoppingCriterion,\n    *,\n    marginal: MarginalFunction = DefaultMarginal(),\n    future_processor: FutureProcessor = FutureProcessor(),\n    batch_size: int = 1,\n    skip_converged: bool = False,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False\n) -&gt; ValuationResult\n</code></pre> <p>Computes semi-values for a given utility function and subset sampler.</p> PARAMETER  DESCRIPTION <code>sampler</code> <p>The subset sampler to use for utility computations.</p> <p> TYPE: <code>PowersetSampler[IndexT]</code> </p> <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>coefficient</code> <p>The semi-value coefficient</p> <p> TYPE: <code>SVCoefficient</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>marginal</code> <p>Marginal function to be used for computing the semivalues</p> <p> TYPE: <code>MarginalFunction</code> DEFAULT: <code>DefaultMarginal()</code> </p> <code>future_processor</code> <p>Additional postprocessing steps required for some algorithms</p> <p> TYPE: <code>FutureProcessor</code> DEFAULT: <code>FutureProcessor()</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>skip_converged</code> <p>Whether to skip marginal evaluations for indices that have already converged. CAUTION: This is only entirely safe if the stopping criterion is MaxUpdates. For any other stopping criterion, the convergence status of indices may change during the computation, or they may be marked as having converged even though in fact the estimated values are far from the true values (e.g. for AbsoluteStandardError, you will probably have to carefully adjust the threshold).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_generic_semivalues(\n    sampler: PowersetSampler[IndexT],\n    u: Utility,\n    coefficient: SVCoefficient,\n    done: StoppingCriterion,\n    *,\n    marginal: MarginalFunction = DefaultMarginal(),\n    future_processor: FutureProcessor = FutureProcessor(),\n    batch_size: int = 1,\n    skip_converged: bool = False,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n) -&gt; ValuationResult:\n    \"\"\"Computes semi-values for a given utility function and subset sampler.\n\n    Args:\n        sampler: The subset sampler to use for utility computations.\n        u: Utility object with model, data, and scoring function.\n        coefficient: The semi-value coefficient\n        done: Stopping criterion.\n        marginal: Marginal function to be used for computing the semivalues\n        future_processor: Additional postprocessing steps required for some algorithms\n        batch_size: Number of marginal evaluations per single parallel job.\n        skip_converged: Whether to skip marginal evaluations for indices that\n            have already converged. **CAUTION**: This is only entirely safe if\n            the stopping criterion is [MaxUpdates][pydvl.value.stopping.MaxUpdates].\n            For any other stopping criterion, the convergence status of indices\n            may change during the computation, or they may be marked as having\n            converged even though in fact the estimated values are far from the\n            true values (e.g. for\n            [AbsoluteStandardError][pydvl.value.stopping.AbsoluteStandardError],\n            you will probably have to carefully adjust the threshold).\n        n_jobs: Number of parallel jobs to use.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    if isinstance(sampler, PermutationSampler) and u.cache is None:\n        log.warning(\n            \"PermutationSampler requires caching to be enabled or computation \"\n            \"will be doubled wrt. a 'direct' implementation of permutation MC\"\n        )\n\n    if batch_size != 1:\n        warnings.warn(\n            \"Parameter `batch_size` is for experimental use and will be\"\n            \" removed in future versions\",\n            DeprecationWarning,\n        )\n\n    result = ValuationResult.zeros(\n        algorithm=f\"semivalue-{str(sampler)}-{coefficient.__name__}\",  # type: ignore\n        indices=u.data.indices,\n        data_names=u.data.data_names,\n    )\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n    u = parallel_backend.put(u)\n    correction = parallel_backend.put(\n        lambda n, k: coefficient(n, k) * sampler.weight(n, k)\n    )\n\n    max_workers = parallel_backend.effective_n_jobs(n_jobs)\n    n_submitted_jobs = 2 * max_workers  # number of jobs in the queue\n\n    sampler_it = iter(sampler)\n    pbar = tqdm(disable=not progress, total=100, unit=\"%\")\n\n    with parallel_backend.executor(\n        max_workers=max_workers, cancel_futures=True\n    ) as executor:\n        pending: set[Future] = set()\n        while True:\n            pbar.n = 100 * done.completion()\n            pbar.refresh()\n\n            completed, pending = wait(pending, timeout=1, return_when=FIRST_COMPLETED)\n            for future in completed:\n                processed_future = future_processor(\n                    future.result()\n                )  # List of tuples or\n                for batch_future in processed_future:\n                    if isinstance(batch_future, list):  # Case when batch size is &gt; 1\n                        for idx, marginal_val in batch_future:\n                            result.update(idx, marginal_val)\n                    else:  # Batch size 1\n                        idx, marginal_val = batch_future\n                        result.update(idx, marginal_val)\n                    if done(result):\n                        return result\n\n            # Ensure that we always have n_submitted_jobs running\n            try:\n                while len(pending) &lt; n_submitted_jobs:\n                    samples = tuple(islice(sampler_it, batch_size))\n                    if len(samples) == 0:\n                        raise StopIteration\n\n                    # Filter out samples for indices that have already converged\n                    filtered_samples = samples\n                    if skip_converged and np.count_nonzero(done.converged) &gt; 0:\n                        # TODO: cloudpickle can't pickle result of `filter` on python 3.8\n                        filtered_samples = tuple(\n                            filter(lambda t: not done.converged[t[0]], samples)\n                        )\n\n                    if filtered_samples:\n                        pending.add(\n                            executor.submit(\n                                marginal,\n                                u=u,\n                                coefficient=correction,\n                                samples=filtered_samples,\n                            )\n                        )\n            except StopIteration:\n                if len(pending) == 0:\n                    return result\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_shapley_semivalues","title":"compute_shapley_semivalues","text":"<pre><code>compute_shapley_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes Shapley values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Shapley coefficient. Use compute_shapley_values for a more flexible interface and additional methods, including TMCS.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_shapley_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    \"\"\"Computes Shapley values for a given utility function.\n\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Shapley coefficient. Use\n    [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values]\n    for a more flexible interface and additional methods, including TMCS.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per single parallel job.\n        n_jobs: Number of parallel jobs to use.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    # HACK: cannot infer return type because of useless IndexT, NameT\n    return compute_generic_semivalues(  # type: ignore\n        sampler_t(u.data.indices, seed=seed),\n        u,\n        shapley_coefficient,\n        done,\n        batch_size=batch_size,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n        config=config,\n        progress=progress,\n    )\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_banzhaf_semivalues","title":"compute_banzhaf_semivalues","text":"<pre><code>compute_banzhaf_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes Banzhaf values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Banzhaf coefficient.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_banzhaf_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    \"\"\"Computes Banzhaf values for a given utility function.\n\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Banzhaf coefficient.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per single parallel job.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    # HACK: cannot infer return type because of useless IndexT, NameT\n    return compute_generic_semivalues(  # type: ignore\n        sampler_t(u.data.indices, seed=seed),\n        u,\n        banzhaf_coefficient,\n        done,\n        batch_size=batch_size,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n        config=config,\n        progress=progress,\n    )\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_msr_banzhaf_semivalues","title":"compute_msr_banzhaf_semivalues","text":"<pre><code>compute_msr_banzhaf_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = MSRSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes MSR sampled Banzhaf values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Banzhaf coefficient and MSR sampling.</p> <p>This algorithm works by sampling random subsets and then evaluating the utility on that subset only once. Based on the evaluation and the subset indices, the MSRFutureProcessor then computes the marginal updates like in the paper (Wang et. al.)<sup>3</sup>. Their approach updates the semivalues for all data points every time a new evaluation is computed. This increases sample efficiency compared to normal Monte Carlo updates.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>MSRSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_msr_banzhaf_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = MSRSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    \"\"\"Computes MSR sampled Banzhaf values for a given utility function.\n\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Banzhaf coefficient and MSR sampling.\n\n    This algorithm works by sampling random subsets and then evaluating the utility\n    on that subset only once. Based on the evaluation and the subset indices,\n    the MSRFutureProcessor then computes the marginal updates like in the paper\n    (Wang et. al.)&lt;sup&gt;&lt;a href=\"wang_data_2023\"&gt;3&lt;/a&gt;&lt;/sup&gt;.\n    Their approach updates the semivalues for all data points every time a new evaluation\n    is computed. This increases sample efficiency compared to normal Monte Carlo updates.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per single parallel job.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n    \"\"\"\n    # HACK: cannot infer return type because of useless IndexT, NameT\n    return compute_generic_semivalues(  # type: ignore\n        sampler_t(u.data.indices, seed=seed),\n        u,\n        always_one_coefficient,\n        done,\n        marginal=RawUtility(),\n        future_processor=MSRFutureProcessor(u),\n        batch_size=batch_size,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n        config=config,\n        progress=progress,\n    )\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_beta_shapley_semivalues","title":"compute_beta_shapley_semivalues","text":"<pre><code>compute_beta_shapley_semivalues(\n    u: Utility,\n    *,\n    alpha: float = 1,\n    beta: float = 1,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes Beta Shapley values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Beta Shapley coefficient.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>alpha</code> <p>Alpha parameter of the Beta distribution.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> <code>beta</code> <p>Beta parameter of the Beta distribution.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per (parallelized) task.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_beta_shapley_semivalues(\n    u: Utility,\n    *,\n    alpha: float = 1,\n    beta: float = 1,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    \"\"\"Computes Beta Shapley values for a given utility function.\n\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Beta Shapley coefficient.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        alpha: Alpha parameter of the Beta distribution.\n        beta: Beta parameter of the Beta distribution.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per (parallelized) task.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    # HACK: cannot infer return type because of useless IndexT, NameT\n    return compute_generic_semivalues(  # type: ignore\n        sampler_t(u.data.indices, seed=seed),\n        u,\n        beta_coefficient(alpha, beta),\n        done,\n        batch_size=batch_size,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n        config=config,\n        progress=progress,\n    )\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_semivalues","title":"compute_semivalues","text":"<pre><code>compute_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    mode: SemiValueMode = SemiValueMode.Shapley,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    seed: Optional[Seed] = None,\n    **kwargs\n) -&gt; ValuationResult\n</code></pre> <p>Convenience entry point for most common semi-value computations.</p> <p>Deprecation warning</p> <p>This method is deprecated and will be replaced in 0.8.0 by the more general implementation of compute_generic_semivalues. Use compute_shapley_semivalues, compute_banzhaf_semivalues, or compute_beta_shapley_semivalues instead.</p> <p>The modes supported with this interface are the following. For greater flexibility use compute_generic_semivalues directly.</p> <ul> <li>SemiValueMode.Shapley:   Shapley values.</li> <li>SemiValueMode.BetaShapley:   Implements the Beta Shapley semi-value as introduced in   (Kwon and Zou, 2022)<sup>1</sup>.   Pass additional keyword arguments <code>alpha</code> and <code>beta</code> to set the   parameters of the Beta distribution (both default to 1).</li> <li>SemiValueMode.Banzhaf: Implements   the Banzhaf semi-value as introduced in (Wang and Jia, 2022)<sup>1</sup>.</li> </ul> <p>See Data valuation for an overview of valuation.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>mode</code> <p>The semi-value mode to use. See SemiValueMode for a list.</p> <p> TYPE: <code>SemiValueMode</code> DEFAULT: <code>Shapley</code> </p> <code>sampler_t</code> <p>The sampler type to use. See sampler for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per (parallelized) task.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments passed to compute_generic_semivalues.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(target=True, deprecated_in=\"0.7.0\", remove_in=\"0.8.0\")\ndef compute_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    mode: SemiValueMode = SemiValueMode.Shapley,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    seed: Optional[Seed] = None,\n    **kwargs,\n) -&gt; ValuationResult:\n    \"\"\"Convenience entry point for most common semi-value computations.\n\n    !!! warning \"Deprecation warning\"\n        This method is deprecated and will be replaced in 0.8.0 by the more\n        general implementation of\n        [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n        Use\n        [compute_shapley_semivalues][pydvl.value.semivalues.compute_shapley_semivalues],\n        [compute_banzhaf_semivalues][pydvl.value.semivalues.compute_banzhaf_semivalues],\n        or\n        [compute_beta_shapley_semivalues][pydvl.value.semivalues.compute_beta_shapley_semivalues]\n        instead.\n\n    The modes supported with this interface are the following. For greater\n    flexibility use\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    directly.\n\n    - [SemiValueMode.Shapley][pydvl.value.semivalues.SemiValueMode]:\n      Shapley values.\n    - [SemiValueMode.BetaShapley][pydvl.value.semivalues.SemiValueMode]:\n      Implements the Beta Shapley semi-value as introduced in\n      (Kwon and Zou, 2022)&lt;sup&gt;&lt;a href=\"#kwon_beta_2022\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n      Pass additional keyword arguments `alpha` and `beta` to set the\n      parameters of the Beta distribution (both default to 1).\n    - [SemiValueMode.Banzhaf][pydvl.value.semivalues.SemiValueMode]: Implements\n      the Banzhaf semi-value as introduced in (Wang and Jia, 2022)&lt;sup&gt;&lt;a\n      href=\"#wang_data_2023\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n\n    See [Data valuation][data-valuation] for an overview of valuation.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        mode: The semi-value mode to use. See\n            [SemiValueMode][pydvl.value.semivalues.SemiValueMode] for a list.\n        sampler_t: The sampler type to use. See [sampler][pydvl.value.sampler]\n            for a list.\n        batch_size: Number of marginal evaluations per (parallelized) task.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        kwargs: Additional keyword arguments passed to\n            [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n    \"\"\"\n    if mode == SemiValueMode.Shapley:\n        coefficient = shapley_coefficient\n    elif mode == SemiValueMode.BetaShapley:\n        alpha = kwargs.pop(\"alpha\", 1)\n        beta = kwargs.pop(\"beta\", 1)\n        coefficient = beta_coefficient(alpha, beta)\n    elif mode == SemiValueMode.Banzhaf:\n        coefficient = banzhaf_coefficient\n    else:\n        raise ValueError(f\"Unknown mode {mode}\")\n    coefficient = cast(SVCoefficient, coefficient)\n\n    # HACK: cannot infer return type because of useless IndexT, NameT\n    return compute_generic_semivalues(  # type: ignore\n        sampler_t(u.data.indices, seed=seed),\n        u,\n        coefficient,\n        done,\n        n_jobs=n_jobs,\n        batch_size=batch_size,\n        **kwargs,\n    )\n</code></pre>"},{"location":"api/pydvl/value/stopping/","title":"Stopping","text":""},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping","title":"pydvl.value.stopping","text":"<p>Stopping criteria for value computations.</p> <p>This module provides a basic set of stopping criteria, like MaxUpdates, MaxTime, or HistoryDeviation among others. These can behave in different ways depending on the context. For example, MaxUpdates limits the number of updates to values, which depending on the algorithm may mean a different number of utility evaluations or imply other computations like solving a linear or quadratic program.</p> <p>Stopping criteria are callables that are evaluated on a ValuationResult and return a Status object. They can be combined using boolean operators.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--how-convergence-is-determined","title":"How convergence is determined","text":"<p>Most stopping criteria keep track of the convergence of each index separately but make global decisions based on the overall convergence of some fraction of all indices. For example, if we have a stopping criterion that checks whether the standard error of 90% of values is below a threshold, then methods will keep updating all indices until 90% of them have converged, irrespective of the quality of the individual estimates, and without freezing updates for indices along the way as values individually attain low standard error.</p> <p>This has some practical implications, because some values do tend to converge sooner than others. For example, assume we use the criterion <code>AbsoluteStandardError(0.02) | MaxUpdates(1000)</code>. Then values close to 0 might be marked as \"converged\" rather quickly because they fulfill the first criterion, say after 20 iterations, despite being poor estimates. Because other indices take much longer to have low standard error and the criterion is a global check, the \"converged\" ones keep being updated and end up being good estimates. In this case, this has been beneficial, but one might not wish for converged values to be updated, if one is sure that the criterion is adequate for individual values.</p> <p>Semi-value methods include a parameter <code>skip_converged</code> that allows to skip the computation of values that have converged. The way to avoid doing this too early is to use a more stringent check, e.g. <code>AbsoluteStandardError(1e-3) | MaxUpdates(1000)</code>. With <code>skip_converged=True</code> this check can still take less time than the first one, despite requiring more iterations for some indices.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--choosing-a-stopping-criterion","title":"Choosing a stopping criterion","text":"<p>The choice of a stopping criterion greatly depends on the algorithm and the context. A safe bet is to combine a MaxUpdates or a MaxTime with a HistoryDeviation or an AbsoluteStandardError. The former will ensure that the computation does not run for too long, while the latter will try to achieve results that are stable enough. Note however that if the threshold is too strict, one will always end up running until a maximum number of iterations or time. Also keep in mind that different values converge at different times, so you might want to use tight thresholds and <code>skip_converged</code> as described above for semi-values.</p> Example <p><pre><code>from pydvl.value import AbsoluteStandardError, MaxUpdates, compute_banzhaf_semivalues\n\nutility = ...  # some utility object\ncriterion = AbsoluteStandardError(threshold=1e-3, burn_in=32) | MaxUpdates(1000)\nvalues = compute_banzhaf_semivalues(\n    utility,\n    criterion,\n    skip_converged=True,  # skip values that have converged (CAREFUL!)\n)\n</code></pre> This will compute the Banzhaf semivalues for <code>utility</code> until either the absolute standard error is below <code>1e-3</code> or <code>1000</code> updates have been performed. The <code>burn_in</code> parameter is used to discard the first <code>32</code> updates from the computation of the standard error. The <code>skip_converged</code> parameter is used to avoid computing more marginals for indices that have converged, which is useful if AbsoluteStandardError is met before MaxUpdates for some indices.</p> <p>Warning</p> <p>Be careful not to reuse the same stopping criterion for different computations. The object has state and will not be reset between calls to value computation methods. If you need to reuse the same criterion, you should create a new instance.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--creating-stopping-criteria","title":"Creating stopping criteria","text":"<p>The easiest way is to declare a function implementing the interface StoppingCriterionCallable and wrap it with make_criterion(). This creates a StoppingCriterion object that can be composed with other stopping criteria.</p> <p>Alternatively, and in particular if reporting of completion is required, one can inherit from this class and implement the abstract methods <code>_check</code> and completion.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--combining-stopping-criteria","title":"Combining stopping criteria","text":"<p>Objects of type StoppingCriterion can be combined with the binary operators <code>&amp;</code> (and), and <code>|</code> (or), following the truth tables of Status. The unary operator <code>~</code> (not) is also supported. See StoppingCriterion for details on how these operations affect the behavior of the stopping criteria.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T. and Jia, R., 2023. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning. In: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics, pp. 6388-6421.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterionCallable","title":"StoppingCriterionCallable","text":"<p>             Bases: <code>Protocol</code></p> <p>Signature for a stopping criterion</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion","title":"StoppingCriterion","text":"<pre><code>StoppingCriterion(modify_result: bool = True)\n</code></pre> <p>             Bases: <code>ABC</code></p> <p>A composable callable object to determine whether a computation must stop.</p> <p>A <code>StoppingCriterion</code> is a callable taking a ValuationResult and returning a Status. It also keeps track of individual convergence of values with converged, and reports the overall completion of the computation with completion.</p> <p>Instances of <code>StoppingCriterion</code> can be composed with the binary operators <code>&amp;</code> (and), and <code>|</code> (or), following the truth tables of Status. The unary operator <code>~</code> (not) is also supported. These boolean operations act according to the following rules:</p> <ul> <li>The results of <code>check()</code> are combined with the operator. See   Status for the truth tables.</li> <li>The results of   converged are combined   with the operator (returning another boolean array).</li> <li>The completion   method returns the min, max, or the complement to 1 of the completions of   the operands, for AND, OR and NOT respectively. This is required for cases   where one of the criteria does not keep track of the convergence of single   values, e.g. MaxUpdates, because   completion by   default returns the mean of the boolean convergence array.</li> </ul>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion--subclassing","title":"Subclassing","text":"<p>Subclassing this class requires implementing a <code>check()</code> method that returns a Status object based on a given ValuationResult. This method should update the attribute <code>_converged</code>, which is a boolean array indicating whether the value for each index has converged. When this does not make sense for a particular stopping criterion, completion should be overridden to provide an overall completion value, since its default implementation attempts to compute the mean of <code>_converged</code>.</p> PARAMETER  DESCRIPTION <code>modify_result</code> <p>If <code>True</code> the status of the input ValuationResult is modified in place after the call.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, modify_result: bool = True):\n    self.modify_result = modify_result\n    self._converged = np.full(0, False)\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.completion","title":"completion","text":"<pre><code>completion() -&gt; float\n</code></pre> <p>Returns a value between 0 and 1 indicating the completion of the computation.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def completion(self) -&gt; float:\n    \"\"\"Returns a value between 0 and 1 indicating the completion of the\n    computation.\n    \"\"\"\n    if self.converged.size == 0:\n        return 0.0\n    return float(np.mean(self.converged).item())\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.AbsoluteStandardError","title":"AbsoluteStandardError","text":"<pre><code>AbsoluteStandardError(\n    threshold: float,\n    fraction: float = 1.0,\n    burn_in: int = 4,\n    modify_result: bool = True,\n)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>Determine convergence based on the standard error of the values.</p> <p>If \\(s_i\\) is the standard error for datum \\(i\\), then this criterion returns Converged if \\(s_i &lt; \\epsilon\\) for all \\(i\\) and a threshold value \\(\\epsilon \\gt 0\\).</p> PARAMETER  DESCRIPTION <code>threshold</code> <p>A value is considered to have converged if the standard error is below this threshold. A way of choosing it is to pick some percentage of the range of the values. For Shapley values this is the difference between the maximum and minimum of the utility function (to see this substitute the maximum and minimum values of the utility into the marginal contribution formula).</p> <p> TYPE: <code>float</code> </p> <code>fraction</code> <p>The fraction of values that must have converged for the criterion to return Converged.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>burn_in</code> <p>The number of iterations to ignore before checking for convergence. This is required because computations typically start with zero variance, as a result of using zeros(). The default is set to an arbitrary minimum which is usually enough but may need to be increased.</p> <p> TYPE: <code>int</code> DEFAULT: <code>4</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(\n    self,\n    threshold: float,\n    fraction: float = 1.0,\n    burn_in: int = 4,\n    modify_result: bool = True,\n):\n    super().__init__(modify_result=modify_result)\n    self.threshold = threshold\n    self.fraction = fraction\n    self.burn_in = burn_in\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.AbsoluteStandardError.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.AbsoluteStandardError.completion","title":"completion","text":"<pre><code>completion() -&gt; float\n</code></pre> <p>Returns a value between 0 and 1 indicating the completion of the computation.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def completion(self) -&gt; float:\n    \"\"\"Returns a value between 0 and 1 indicating the completion of the\n    computation.\n    \"\"\"\n    if self.converged.size == 0:\n        return 0.0\n    return float(np.mean(self.converged).item())\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.AbsoluteStandardError.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxChecks","title":"MaxChecks","text":"<pre><code>MaxChecks(n_checks: Optional[int], modify_result: bool = True)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate as soon as the number of checks exceeds the threshold.</p> <p>A \"check\" is one call to the criterion.</p> PARAMETER  DESCRIPTION <code>n_checks</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_checks: Optional[int], modify_result: bool = True):\n    super().__init__(modify_result=modify_result)\n    if n_checks is not None and n_checks &lt; 1:\n        raise ValueError(\"n_iterations must be at least 1 or None\")\n    self.n_checks = n_checks\n    self._count = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxChecks.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxChecks.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxUpdates","title":"MaxUpdates","text":"<pre><code>MaxUpdates(n_updates: Optional[int], modify_result: bool = True)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate if any number of value updates exceeds or equals the given threshold.</p> <p>Note</p> <p>If you want to ensure that all values have been updated, you probably want MinUpdates instead.</p> <p>This checks the <code>counts</code> field of a ValuationResult, i.e. the number of times that each index has been updated. For powerset samplers, the maximum of this number coincides with the maximum number of subsets sampled. For permutation samplers, it coincides with the number of permutations sampled.</p> PARAMETER  DESCRIPTION <code>n_updates</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_updates: Optional[int], modify_result: bool = True):\n    super().__init__(modify_result=modify_result)\n    if n_updates is not None and n_updates &lt; 1:\n        raise ValueError(\"n_updates must be at least 1 or None\")\n    self.n_updates = n_updates\n    self.last_max = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxUpdates.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxUpdates.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MinUpdates","title":"MinUpdates","text":"<pre><code>MinUpdates(n_updates: Optional[int], modify_result: bool = True)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate as soon as all value updates exceed or equal the given threshold.</p> <p>This checks the <code>counts</code> field of a ValuationResult, i.e. the number of times that each index has been updated. For powerset samplers, the minimum of this number is a lower bound for the number of subsets sampled. For permutation samplers, it lower-bounds the amount of permutations sampled.</p> PARAMETER  DESCRIPTION <code>n_updates</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_updates: Optional[int], modify_result: bool = True):\n    super().__init__(modify_result=modify_result)\n    self.n_updates = n_updates\n    self.last_min = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MinUpdates.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MinUpdates.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxTime","title":"MaxTime","text":"<pre><code>MaxTime(seconds: Optional[float], modify_result: bool = True)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate if the computation time exceeds the given number of seconds.</p> <p>Checks the elapsed time since construction</p> PARAMETER  DESCRIPTION <code>seconds</code> <p>Threshold: The computation is terminated if the elapsed time between object construction and a _check exceeds this value. If <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[float]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, seconds: Optional[float], modify_result: bool = True):\n    super().__init__(modify_result=modify_result)\n    self.max_seconds = seconds or np.inf\n    if self.max_seconds &lt;= 0:\n        raise ValueError(\"Number of seconds for MaxTime must be positive or None\")\n    self.start = time()\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxTime.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxTime.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.HistoryDeviation","title":"HistoryDeviation","text":"<pre><code>HistoryDeviation(\n    n_steps: int,\n    rtol: float,\n    pin_converged: bool = True,\n    modify_result: bool = True,\n)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>A simple check for relative distance to a previous step in the computation.</p> <p>The method used by (Ghorbani and Zou, 2019)<sup>1</sup> computes the relative distances between the current values \\(v_i^t\\) and the values at the previous checkpoint \\(v_i^{t-\\tau}\\). If the sum is below a given threshold, the computation is terminated.</p> \\[\\sum_{i=1}^n \\frac{\\left| v_i^t - v_i^{t-\\tau} \\right|}{v_i^t} &lt; \\epsilon.\\] <p>When the denominator is zero, the summand is set to the value of \\(v_i^{ t-\\tau}\\).</p> <p>This implementation is slightly generalised to allow for different number of updates to individual indices, as happens with powerset samplers instead of permutations. Every subset of indices that is found to converge can be pinned to that state. Once all indices have converged the method has converged.</p> <p>Warning</p> <p>This criterion is meant for the reproduction of the results in the paper, but we do not recommend using it in practice.</p> PARAMETER  DESCRIPTION <code>n_steps</code> <p>Checkpoint values every so many updates and use these saved values to compare.</p> <p> TYPE: <code>int</code> </p> <code>rtol</code> <p>Relative tolerance for convergence (\\(\\epsilon\\) in the formula).</p> <p> TYPE: <code>float</code> </p> <code>pin_converged</code> <p>If <code>True</code>, once an index has converged, it is pinned</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(\n    self,\n    n_steps: int,\n    rtol: float,\n    pin_converged: bool = True,\n    modify_result: bool = True,\n):\n    super().__init__(modify_result=modify_result)\n    if n_steps &lt; 1:\n        raise ValueError(\"n_steps must be at least 1\")\n    if rtol &lt;= 0 or rtol &gt;= 1:\n        raise ValueError(\"rtol must be in (0, 1)\")\n\n    self.n_steps = n_steps\n    self.rtol = rtol\n    self.update_op = np.logical_or if pin_converged else np.logical_and\n    self._memory = None  # type: ignore\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.HistoryDeviation.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.HistoryDeviation.completion","title":"completion","text":"<pre><code>completion() -&gt; float\n</code></pre> <p>Returns a value between 0 and 1 indicating the completion of the computation.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def completion(self) -&gt; float:\n    \"\"\"Returns a value between 0 and 1 indicating the completion of the\n    computation.\n    \"\"\"\n    if self.converged.size == 0:\n        return 0.0\n    return float(np.mean(self.converged).item())\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.HistoryDeviation.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.RankCorrelation","title":"RankCorrelation","text":"<pre><code>RankCorrelation(rtol: float, burn_in: int, modify_result: bool = True)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>A check for stability of Spearman correlation between checks.</p> <p>When the change in rank correlation between two successive iterations is below a given threshold, the computation is terminated. The criterion computes the Spearman correlation between two successive iterations. The Spearman correlation uses the ordering indices of the given values and correlates them. This means it focuses on the order of the elements instead of their exact values. If the order stops changing (meaning the Banzhaf semivalues estimates converge), the criterion stops the algorithm.</p> <p>This criterion is used in (Wang et. al.)<sup>2</sup>.</p> PARAMETER  DESCRIPTION <code>rtol</code> <p>Relative tolerance for convergence (\\(\\epsilon\\) in the formula)</p> <p> TYPE: <code>float</code> </p> <code>modify_result</code> <p>If <code>True</code>, the status of the input ValuationResult is modified in place after the call.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>burn_in</code> <p>The minimum number of iterations before checking for convergence. This is required because the first correlation is meaningless.</p> <p> TYPE: <code>int</code> </p> <p>Added in 0.9.0</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(\n    self,\n    rtol: float,\n    burn_in: int,\n    modify_result: bool = True,\n):\n    super().__init__(modify_result=modify_result)\n    if rtol &lt;= 0 or rtol &gt;= 1:\n        raise ValueError(\"rtol must be in (0, 1)\")\n    self.rtol = rtol\n    self.burn_in = burn_in\n    self._memory: NDArray[np.float_] | None = None\n    self._corr = 0.0\n    self._completion = 0.0\n    self._iterations = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.RankCorrelation.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.RankCorrelation.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.make_criterion","title":"make_criterion","text":"<pre><code>make_criterion(\n    fun: StoppingCriterionCallable,\n    converged: Callable[[], NDArray[bool_]] | None = None,\n    completion: Callable[[], float] | None = None,\n    name: str | None = None,\n) -&gt; Type[StoppingCriterion]\n</code></pre> <p>Create a new StoppingCriterion from a function. Use this to enable simpler functions to be composed with bitwise operators</p> PARAMETER  DESCRIPTION <code>fun</code> <p>The callable to wrap.</p> <p> TYPE: <code>StoppingCriterionCallable</code> </p> <code>converged</code> <p>A callable that returns a boolean array indicating what values have converged.</p> <p> TYPE: <code>Callable[[], NDArray[bool_]] | None</code> DEFAULT: <code>None</code> </p> <code>completion</code> <p>A callable that returns a value between 0 and 1 indicating the rate of completion of the computation. If not provided, the fraction of converged values is used.</p> <p> TYPE: <code>Callable[[], float] | None</code> DEFAULT: <code>None</code> </p> <code>name</code> <p>The name of the new criterion. If <code>None</code>, the <code>__name__</code> of the function is used.</p> <p> TYPE: <code>str | None</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Type[StoppingCriterion]</code> <p>A new subclass of StoppingCriterion.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def make_criterion(\n    fun: StoppingCriterionCallable,\n    converged: Callable[[], NDArray[np.bool_]] | None = None,\n    completion: Callable[[], float] | None = None,\n    name: str | None = None,\n) -&gt; Type[StoppingCriterion]:\n    \"\"\"Create a new [StoppingCriterion][pydvl.value.stopping.StoppingCriterion] from a function.\n    Use this to enable simpler functions to be composed with bitwise operators\n\n    Args:\n        fun: The callable to wrap.\n        converged: A callable that returns a boolean array indicating what\n            values have converged.\n        completion: A callable that returns a value between 0 and 1 indicating\n            the rate of completion of the computation. If not provided, the fraction\n            of converged values is used.\n        name: The name of the new criterion. If `None`, the `__name__` of\n            the function is used.\n\n    Returns:\n        A new subclass of [StoppingCriterion][pydvl.value.stopping.StoppingCriterion].\n    \"\"\"\n\n    class WrappedCriterion(StoppingCriterion):\n        def __init__(self, modify_result: bool = True):\n            super().__init__(modify_result=modify_result)\n            self._name = name or getattr(fun, \"__name__\", \"WrappedCriterion\")\n\n        def _check(self, result: ValuationResult) -&gt; Status:\n            return fun(result)\n\n        @property\n        def converged(self) -&gt; NDArray[np.bool_]:\n            if converged is None:\n                return super().converged\n            return converged()\n\n        def __str__(self):\n            return self._name\n\n        def completion(self) -&gt; float:\n            if completion is None:\n                return super().completion()\n            return completion()\n\n    return WrappedCriterion\n</code></pre>"},{"location":"api/pydvl/value/least_core/","title":"Least core","text":""},{"location":"api/pydvl/value/least_core/#pydvl.value.least_core","title":"pydvl.value.least_core","text":"<p>New in version 0.4.0</p> <p>This package holds all routines for the computation of Least Core data values.</p> <p>Please refer to Data valuation for an overview.</p> <p>In addition to the standard interface via compute_least_core_values(), because computing the Least Core values requires the solution of a linear and a quadratic problem after computing all the utility values, there is the possibility of performing each step separately. This is useful when running multiple experiments: use lc_prepare_problem() or mclc_prepare_problem() to prepare a list of problems to solve, then solve them in parallel with lc_solve_problems().</p> <p>Note that mclc_prepare_problem() is parallelized itself, so preparing the problems should be done in sequence in this case. The solution of the linear systems can then be done in parallel.</p>"},{"location":"api/pydvl/value/least_core/#pydvl.value.least_core.LeastCoreMode","title":"LeastCoreMode","text":"<p>             Bases: <code>Enum</code></p> <p>Available Least Core algorithms.</p>"},{"location":"api/pydvl/value/least_core/#pydvl.value.least_core.compute_least_core_values","title":"compute_least_core_values","text":"<pre><code>compute_least_core_values(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    n_iterations: Optional[int] = None,\n    mode: LeastCoreMode = LeastCoreMode.MonteCarlo,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = False,\n    **kwargs\n) -&gt; ValuationResult\n</code></pre> <p>Umbrella method to compute Least Core values with any of the available algorithms.</p> <p>See Data valuation for an overview.</p> <p>The following algorithms are available. Note that the exact method can only work with very small datasets and is thus intended only for testing.</p> <ul> <li><code>exact</code>: uses the complete powerset of the training set for the constraints   combinatorial_exact_shapley().</li> <li><code>montecarlo</code>:  uses the approximate Monte Carlo Least Core algorithm.   Implemented in montecarlo_least_core().</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_jobs</code> <p>Number of jobs to run in parallel. Only used for Monte Carlo Least Core.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_iterations</code> <p>Number of subsets to sample and evaluate the utility on. Only used for Monte Carlo Least Core.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>Algorithm to use. See LeastCoreMode for available options.</p> <p> TYPE: <code>LeastCoreMode</code> DEFAULT: <code>MonteCarlo</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>solver_options</code> <p>Optional dictionary of options passed to the solvers.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the computed values.</p> <p>New in version 0.5.0</p> Source code in <code>src/pydvl/value/least_core/__init__.py</code> <pre><code>def compute_least_core_values(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    n_iterations: Optional[int] = None,\n    mode: LeastCoreMode = LeastCoreMode.MonteCarlo,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = False,\n    **kwargs,\n) -&gt; ValuationResult:\n    \"\"\"Umbrella method to compute Least Core values with any of the available\n    algorithms.\n\n    See [Data valuation][data-valuation] for an overview.\n\n    The following algorithms are available. Note that the exact method can only\n    work with very small datasets and is thus intended only for testing.\n\n    - `exact`: uses the complete powerset of the training set for the constraints\n      [combinatorial_exact_shapley()][pydvl.value.shapley.naive.combinatorial_exact_shapley].\n    - `montecarlo`:  uses the approximate Monte Carlo Least Core algorithm.\n      Implemented in [montecarlo_least_core()][pydvl.value.least_core.montecarlo.montecarlo_least_core].\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_jobs: Number of jobs to run in parallel. Only used for Monte Carlo\n            Least Core.\n        n_iterations: Number of subsets to sample and evaluate the utility on.\n            Only used for Monte Carlo Least Core.\n        mode: Algorithm to use. See\n            [LeastCoreMode][pydvl.value.least_core.LeastCoreMode] for available\n            options.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Optional dictionary of options passed to the solvers.\n\n    Returns:\n        Object with the computed values.\n\n    !!! tip \"New in version 0.5.0\"\n    \"\"\"\n\n    if mode == LeastCoreMode.MonteCarlo:\n        # TODO fix progress showing in remote case\n        progress = False\n        if n_iterations is None:\n            raise ValueError(\"n_iterations cannot be None for Monte Carlo Least Core\")\n        return montecarlo_least_core(  # type: ignore\n            u=u,\n            n_iterations=n_iterations,\n            n_jobs=n_jobs,\n            progress=progress,\n            non_negative_subsidy=non_negative_subsidy,\n            solver_options=solver_options,\n            **kwargs,\n        )\n    elif mode == LeastCoreMode.Exact:\n        return exact_least_core(\n            u=u,\n            progress=progress,\n            non_negative_subsidy=non_negative_subsidy,\n            solver_options=solver_options,\n        )\n\n    raise ValueError(f\"Invalid value encountered in {mode=}\")\n</code></pre>"},{"location":"api/pydvl/value/least_core/common/","title":"Common","text":""},{"location":"api/pydvl/value/least_core/common/#pydvl.value.least_core.common","title":"pydvl.value.least_core.common","text":""},{"location":"api/pydvl/value/least_core/common/#pydvl.value.least_core.common.lc_solve_problem","title":"lc_solve_problem","text":"<pre><code>lc_solve_problem(\n    problem: LeastCoreProblem,\n    *,\n    u: Utility,\n    algorithm: str,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None\n) -&gt; ValuationResult\n</code></pre> <p>Solves a linear problem as prepared by mclc_prepare_problem(). Useful for parallel execution of multiple experiments by running this as a remote task.</p> <p>See exact_least_core() or montecarlo_least_core() for argument descriptions.</p> Source code in <code>src/pydvl/value/least_core/common.py</code> <pre><code>def lc_solve_problem(\n    problem: LeastCoreProblem,\n    *,\n    u: Utility,\n    algorithm: str,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n) -&gt; ValuationResult:\n    \"\"\"Solves a linear problem as prepared by\n    [mclc_prepare_problem()][pydvl.value.least_core.montecarlo.mclc_prepare_problem].\n    Useful for parallel execution of multiple experiments by running this as a\n    remote task.\n\n    See [exact_least_core()][pydvl.value.least_core.naive.exact_least_core] or\n    [montecarlo_least_core()][pydvl.value.least_core.montecarlo.montecarlo_least_core] for\n    argument descriptions.\n    \"\"\"\n    n = len(u.data)\n\n    if np.any(np.isnan(problem.utility_values)):\n        warnings.warn(\n            f\"Calculation returned \"\n            f\"{np.sum(np.isnan(problem.utility_values))} NaN \"\n            f\"values out of {problem.utility_values.size}\",\n            RuntimeWarning,\n        )\n\n    if solver_options is None:\n        solver_options = {}\n\n    if \"solver\" not in solver_options:\n        solver_options[\"solver\"] = cp.SCS\n\n    if \"max_iters\" not in solver_options and solver_options[\"solver\"] == cp.SCS:\n        solver_options[\"max_iters\"] = 10000\n\n    logger.debug(\"Removing possible duplicate values in lower bound array\")\n    b_lb = problem.utility_values\n    A_lb, unique_indices = np.unique(problem.A_lb, return_index=True, axis=0)\n    b_lb = b_lb[unique_indices]\n\n    logger.debug(\"Building equality constraint\")\n    A_eq = np.ones((1, n))\n    # We might have already computed the total utility one or more times.\n    # This is the index of the row(s) in A_lb with all ones.\n    total_utility_indices = np.where(A_lb.sum(axis=1) == n)[0]\n    if len(total_utility_indices) == 0:\n        b_eq = np.array([u(u.data.indices)])\n    else:\n        b_eq = b_lb[total_utility_indices]\n        # Remove the row(s) corresponding to the total utility\n        # from the lower bound constraints\n        # because given the equality constraint\n        # it is the same as using the constraint e &gt;= 0\n        # (i.e. setting non_negative_subsidy = True).\n        mask: NDArray[np.bool_] = np.ones_like(b_lb, dtype=bool)\n        mask[total_utility_indices] = False\n        b_lb = b_lb[mask]\n        A_lb = A_lb[mask]\n\n    # Remove the row(s) corresponding to the empty subset\n    # because, given u(\u2205) = (which is almost always the case,\n    # it is the same as using the constraint e &gt;= 0\n    # (i.e. setting non_negative_subsidy = True).\n    emptyset_utility_indices = np.where(A_lb.sum(axis=1) == 0)[0]\n    if len(emptyset_utility_indices) &gt; 0:\n        mask = np.ones_like(b_lb, dtype=bool)\n        mask[emptyset_utility_indices] = False\n        b_lb = b_lb[mask]\n        A_lb = A_lb[mask]\n\n    _, subsidy = _solve_least_core_linear_program(\n        A_eq=A_eq,\n        b_eq=b_eq,\n        A_lb=A_lb,\n        b_lb=b_lb,\n        non_negative_subsidy=non_negative_subsidy,\n        solver_options=solver_options,\n    )\n\n    values: Optional[NDArray[np.float_]]\n\n    if subsidy is None:\n        logger.debug(\"No values were found\")\n        status = Status.Failed\n        values = np.empty(n)\n        values[:] = np.nan\n        subsidy = np.nan\n    else:\n        values = _solve_egalitarian_least_core_quadratic_program(\n            subsidy,\n            A_eq=A_eq,\n            b_eq=b_eq,\n            A_lb=A_lb,\n            b_lb=b_lb,\n            solver_options=solver_options,\n        )\n\n        if values is None:\n            logger.debug(\"No values were found\")\n            status = Status.Failed\n            values = np.empty(n)\n            values[:] = np.nan\n            subsidy = np.nan\n        else:\n            status = Status.Converged\n\n    return ValuationResult(\n        algorithm=algorithm,\n        status=status,\n        values=values,\n        subsidy=subsidy,\n        stderr=None,\n        data_names=u.data.data_names,\n    )\n</code></pre>"},{"location":"api/pydvl/value/least_core/common/#pydvl.value.least_core.common.lc_solve_problems","title":"lc_solve_problems","text":"<pre><code>lc_solve_problems(\n    problems: Sequence[LeastCoreProblem],\n    u: Utility,\n    algorithm: str,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    n_jobs: int = 1,\n    non_negative_subsidy: bool = True,\n    solver_options: Optional[dict] = None,\n    **options\n) -&gt; List[ValuationResult]\n</code></pre> <p>Solves a list of linear problems in parallel.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility.</p> <p> TYPE: <code>Utility</code> </p> <code>problems</code> <p>Least Core problems to solve, as returned by mclc_prepare_problem().</p> <p> TYPE: <code>Sequence[LeastCoreProblem]</code> </p> <code>algorithm</code> <p>Name of the valuation algorithm.</p> <p> TYPE: <code>str</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to run.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>solver_options</code> <p>Additional options to pass to the solver.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>List[ValuationResult]</code> <p>List of solutions.</p> Source code in <code>src/pydvl/value/least_core/common.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef lc_solve_problems(\n    problems: Sequence[LeastCoreProblem],\n    u: Utility,\n    algorithm: str,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    n_jobs: int = 1,\n    non_negative_subsidy: bool = True,\n    solver_options: Optional[dict] = None,\n    **options,\n) -&gt; List[ValuationResult]:\n    \"\"\"Solves a list of linear problems in parallel.\n\n    Args:\n        u: Utility.\n        problems: Least Core problems to solve, as returned by\n            [mclc_prepare_problem()][pydvl.value.least_core.montecarlo.mclc_prepare_problem].\n        algorithm: Name of the valuation algorithm.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        n_jobs: Number of parallel jobs to run.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Additional options to pass to the solver.\n\n    Returns:\n        List of solutions.\n    \"\"\"\n\n    def _map_func(\n        problems: List[LeastCoreProblem], *args, **kwargs\n    ) -&gt; List[ValuationResult]:\n        return [lc_solve_problem(p, *args, **kwargs) for p in problems]\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    map_reduce_job: MapReduceJob[\n        \"LeastCoreProblem\", \"List[ValuationResult]\"\n    ] = MapReduceJob(\n        inputs=problems,\n        map_func=_map_func,\n        map_kwargs=dict(\n            u=u,\n            algorithm=algorithm,\n            non_negative_subsidy=non_negative_subsidy,\n            solver_options=solver_options,\n            **options,\n        ),\n        reduce_func=lambda x: list(itertools.chain(*x)),\n        parallel_backend=parallel_backend,\n        n_jobs=n_jobs,\n    )\n    solutions = map_reduce_job()\n\n    return solutions\n</code></pre>"},{"location":"api/pydvl/value/least_core/montecarlo/","title":"Montecarlo","text":""},{"location":"api/pydvl/value/least_core/montecarlo/#pydvl.value.least_core.montecarlo","title":"pydvl.value.least_core.montecarlo","text":""},{"location":"api/pydvl/value/least_core/montecarlo/#pydvl.value.least_core.montecarlo.montecarlo_least_core","title":"montecarlo_least_core","text":"<pre><code>montecarlo_least_core(\n    u: Utility,\n    n_iterations: int,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes approximate Least Core values using a Monte Carlo approach.</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\ &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp; , \\forall S \\in \\{S_1, S_2, \\dots, S_m \\overset{\\mathrm{iid}}{\\sim} U(2^N) \\} \\end{array} \\] <p>Where:</p> <ul> <li>\\(U(2^N)\\) is the uniform distribution over the powerset of \\(N\\).</li> <li>\\(m\\) is the number of subsets that will be sampled and whose utility will   be computed and used to compute the data values.</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_iterations</code> <p>total number of iterations to use</p> <p> TYPE: <code>int</code> </p> <code>n_jobs</code> <p>number of jobs across which to distribute the computation</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>solver_options</code> <p>Dictionary of options that will be used to select a solver and to configure it. Refer to cvxpy's documentation for all possible options.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, shows a tqdm progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values and the least core value.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/least_core/montecarlo.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef montecarlo_least_core(\n    u: Utility,\n    n_iterations: int,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    r\"\"\"Computes approximate Least Core values using a Monte Carlo approach.\n\n    $$\n    \\begin{array}{lll}\n    \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\\n    \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\\n    &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp; ,\n    \\forall S \\in \\{S_1, S_2, \\dots, S_m \\overset{\\mathrm{iid}}{\\sim} U(2^N) \\}\n    \\end{array}\n    $$\n\n    Where:\n\n    * $U(2^N)$ is the uniform distribution over the powerset of $N$.\n    * $m$ is the number of subsets that will be sampled and whose utility will\n      be computed and used to compute the data values.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_iterations: total number of iterations to use\n        n_jobs: number of jobs across which to distribute the computation\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Dictionary of options that will be used to select a solver\n            and to configure it. Refer to [cvxpy's\n            documentation](https://www.cvxpy.org/tutorial/advanced/index.html#setting-solver-options)\n            for all possible options.\n        progress: If True, shows a tqdm progress bar\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Object with the data values and the least core value.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    problem = mclc_prepare_problem(\n        u,\n        n_iterations,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n        config=config,\n        progress=progress,\n        seed=seed,\n    )\n    return lc_solve_problem(\n        problem,\n        u=u,\n        algorithm=\"montecarlo_least_core\",\n        non_negative_subsidy=non_negative_subsidy,\n        solver_options=solver_options,\n    )\n</code></pre>"},{"location":"api/pydvl/value/least_core/montecarlo/#pydvl.value.least_core.montecarlo.mclc_prepare_problem","title":"mclc_prepare_problem","text":"<pre><code>mclc_prepare_problem(\n    u: Utility,\n    n_iterations: int,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; LeastCoreProblem\n</code></pre> <p>Prepares a linear problem by sampling subsets of the data. Use this to separate the problem preparation from the solving with lc_solve_problem(). Useful for parallel execution of multiple experiments.</p> <p>See montecarlo_least_core for argument descriptions.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/least_core/montecarlo.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef mclc_prepare_problem(\n    u: Utility,\n    n_iterations: int,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; LeastCoreProblem:\n    \"\"\"Prepares a linear problem by sampling subsets of the data. Use this to\n    separate the problem preparation from the solving with\n    [lc_solve_problem()][pydvl.value.least_core.common.lc_solve_problem]. Useful\n    for parallel execution of multiple experiments.\n\n    See\n    [montecarlo_least_core][pydvl.value.least_core.montecarlo.montecarlo_least_core]\n    for argument descriptions.\n\n    !!! note \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    n = len(u.data)\n\n    if n_iterations &lt; n:\n        warnings.warn(\n            f\"Number of iterations '{n_iterations}' is smaller the size of the dataset '{n}'. \"\n            f\"This is not optimal because in the worst case we need at least '{n}' constraints \"\n            \"to satisfy the individual rationality condition.\"\n        )\n\n    if n_iterations &gt; 2**n:\n        warnings.warn(\n            f\"Passed n_iterations is greater than the number subsets! \"\n            f\"Setting it to 2^{n}\",\n            RuntimeWarning,\n        )\n        n_iterations = 2**n\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    iterations_per_job = max(\n        1, n_iterations // parallel_backend.effective_n_jobs(n_jobs)\n    )\n\n    map_reduce_job: MapReduceJob[\"Utility\", \"LeastCoreProblem\"] = MapReduceJob(\n        inputs=u,\n        map_func=_montecarlo_least_core,\n        reduce_func=_reduce_func,\n        map_kwargs=dict(n_iterations=iterations_per_job, progress=progress),\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n    )\n\n    return map_reduce_job(seed=seed)\n</code></pre>"},{"location":"api/pydvl/value/least_core/naive/","title":"Naive","text":""},{"location":"api/pydvl/value/least_core/naive/#pydvl.value.least_core.naive","title":"pydvl.value.least_core.naive","text":""},{"location":"api/pydvl/value/least_core/naive/#pydvl.value.least_core.naive.exact_least_core","title":"exact_least_core","text":"<pre><code>exact_least_core(\n    u: Utility,\n    *,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = True\n) -&gt; ValuationResult\n</code></pre> <p>Computes the exact Least Core values.</p> <p>Note</p> <p>If the training set contains more than 20 instances a warning is printed because the computation is very expensive. This method is mostly used for internal testing and simple use cases. Please refer to the Monte Carlo method for practical applications.</p> <p>The least core is the solution to the following Linear Programming problem:</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\ &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp;, \\forall S \\subseteq N \\\\ \\end{array} \\] <p>Where \\(N = \\{1, 2, \\dots, n\\}\\) are the training set's indices.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>solver_options</code> <p>Dictionary of options that will be used to select a solver and to configure it. Refer to the cvxpy's documentation for all possible options.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, shows a tqdm progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values and the least core value.</p> Source code in <code>src/pydvl/value/least_core/naive.py</code> <pre><code>def exact_least_core(\n    u: Utility,\n    *,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = True,\n) -&gt; ValuationResult:\n    r\"\"\"Computes the exact Least Core values.\n\n    !!! Note\n        If the training set contains more than 20 instances a warning is printed\n        because the computation is very expensive. This method is mostly used for\n        internal testing and simple use cases. Please refer to the\n        [Monte Carlo method][pydvl.value.least_core.montecarlo.montecarlo_least_core]\n        for practical applications.\n\n    The least core is the solution to the following Linear Programming problem:\n\n    $$\n    \\begin{array}{lll}\n    \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\\n    \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\\n    &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp;, \\forall S \\subseteq N \\\\\n    \\end{array}\n    $$\n\n    Where $N = \\{1, 2, \\dots, n\\}$ are the training set's indices.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Dictionary of options that will be used to select a solver\n            and to configure it. Refer to the [cvxpy's\n            documentation](https://www.cvxpy.org/tutorial/advanced/index.html#setting-solver-options)\n            for all possible options.\n        progress: If True, shows a tqdm progress bar\n\n    Returns:\n        Object with the data values and the least core value.\n    \"\"\"\n    n = len(u.data)\n    if n &gt; 20:  # Arbitrary choice, will depend on time required, caching, etc.\n        warnings.warn(f\"Large dataset! Computation requires 2^{n} calls to model.fit()\")\n\n    problem = lc_prepare_problem(u, progress=progress)\n    return lc_solve_problem(\n        problem=problem,\n        u=u,\n        algorithm=\"exact_least_core\",\n        non_negative_subsidy=non_negative_subsidy,\n        solver_options=solver_options,\n    )\n</code></pre>"},{"location":"api/pydvl/value/least_core/naive/#pydvl.value.least_core.naive.lc_prepare_problem","title":"lc_prepare_problem","text":"<pre><code>lc_prepare_problem(u: Utility, progress: bool = False) -&gt; LeastCoreProblem\n</code></pre> <p>Prepares a linear problem with all subsets of the data Use this to separate the problem preparation from the solving with lc_solve_problem(). Useful for parallel execution of multiple experiments.</p> <p>See exact_least_core() for argument descriptions.</p> Source code in <code>src/pydvl/value/least_core/naive.py</code> <pre><code>def lc_prepare_problem(u: Utility, progress: bool = False) -&gt; LeastCoreProblem:\n    \"\"\"Prepares a linear problem with all subsets of the data\n    Use this to separate the problem preparation from the solving with\n    [lc_solve_problem()][pydvl.value.least_core.common.lc_solve_problem]. Useful for\n    parallel execution of multiple experiments.\n\n    See [exact_least_core()][pydvl.value.least_core.naive.exact_least_core] for argument\n    descriptions.\n    \"\"\"\n    n = len(u.data)\n\n    logger.debug(\"Building vectors and matrices for linear programming problem\")\n    powerset_size = 2**n\n    A_lb = np.zeros((powerset_size, n))\n\n    logger.debug(\"Iterating over all subsets\")\n    utility_values = np.zeros(powerset_size)\n    for i, subset in enumerate(  # type: ignore\n        tqdm(\n            powerset(u.data.indices),\n            disable=not progress,\n            total=powerset_size - 1,\n            position=0,\n        )\n    ):\n        indices: NDArray[np.bool_] = np.zeros(n, dtype=bool)\n        indices[list(subset)] = True\n        A_lb[i, indices] = 1\n        utility_values[i] = u(subset)  # type: ignore\n\n    return LeastCoreProblem(utility_values, A_lb)\n</code></pre>"},{"location":"api/pydvl/value/loo/","title":"Loo","text":""},{"location":"api/pydvl/value/loo/#pydvl.value.loo","title":"pydvl.value.loo","text":""},{"location":"api/pydvl/value/loo/loo/","title":"Loo","text":""},{"location":"api/pydvl/value/loo/loo/#pydvl.value.loo.loo","title":"pydvl.value.loo.loo","text":""},{"location":"api/pydvl/value/loo/loo/#pydvl.value.loo.loo.compute_loo","title":"compute_loo","text":"<pre><code>compute_loo(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = True\n) -&gt; ValuationResult\n</code></pre> <p>Computes leave one out value:</p> \\[v(i) = u(D) - u(D \\setminus \\{i\\}) \\] PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>If True, display a progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, display a progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.7.0</p> <p>Renamed from <code>naive_loo</code> and added parallel computation.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/loo/loo.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_loo(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = True,\n) -&gt; ValuationResult:\n    r\"\"\"Computes leave one out value:\n\n    $$v(i) = u(D) - u(D \\setminus \\{i\\}) $$\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        progress: If True, display a progress bar\n        n_jobs: Number of parallel jobs to use\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: If True, display a progress bar\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"New in version 0.7.0\"\n        Renamed from `naive_loo` and added parallel computation.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    if len(u.data) &lt; 3:\n        raise ValueError(\"Dataset must have at least 2 elements\")\n\n    result = ValuationResult.zeros(\n        algorithm=\"loo\",\n        indices=u.data.indices,\n        data_names=u.data.data_names,\n    )\n\n    all_indices = set(u.data.indices)\n    total_utility = u(u.data.indices)\n\n    def fun(idx: int) -&gt; tuple[int, float]:\n        return idx, total_utility - u(all_indices.difference({idx}))\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n    max_workers = parallel_backend.effective_n_jobs(n_jobs)\n    n_submitted_jobs = 2 * max_workers  # number of jobs in the queue\n\n    # NOTE: this could be done with a simple executor.map(), but we want to\n    # display a progress bar\n\n    with parallel_backend.executor(\n        max_workers=max_workers, cancel_futures=True\n    ) as executor:\n        pending: set[Future] = set()\n        index_it = iter(u.data.indices)\n\n        pbar = tqdm(disable=not progress, total=100, unit=\"%\")\n        while True:\n            pbar.n = 100 * sum(result.counts) / len(u.data)\n            pbar.refresh()\n            completed, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED)\n            for future in completed:\n                idx, marginal = future.result()\n                result.update(idx, marginal)\n\n            # Ensure that we always have n_submitted_jobs running\n            try:\n                for _ in range(n_submitted_jobs - len(pending)):\n                    pending.add(executor.submit(fun, next(index_it)))\n            except StopIteration:\n                if len(pending) == 0:\n                    return result\n</code></pre>"},{"location":"api/pydvl/value/oob/","title":"Oob","text":""},{"location":"api/pydvl/value/oob/#pydvl.value.oob","title":"pydvl.value.oob","text":""},{"location":"api/pydvl/value/oob/oob/","title":"Oob","text":""},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob","title":"pydvl.value.oob.oob","text":""},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob--references","title":"References","text":"<ol> <li> <p>Kwon et al. Data-OOB: Out-of-bag Estimate as a Simple and Efficient Data Value. In: Published at ICML 2023\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.compute_data_oob","title":"compute_data_oob","text":"<pre><code>compute_data_oob(\n    u: Utility,\n    *,\n    n_est: int = 10,\n    max_samples: float = 0.8,\n    loss: Optional[LossFunction] = None,\n    n_jobs: Optional[int] = None,\n    seed: Optional[Seed] = None,\n    progress: bool = False\n) -&gt; ValuationResult\n</code></pre> <p>Computes Data out of bag values</p> <p>This implements the method described in (Kwon and Zou, 2023)<sup>1</sup>. It fits several base estimators provided through u.model through a bagging process. The point value corresponds to the average loss of estimators which were not fit on it.</p> <p>\\(w_{bj}\\in Z\\) is the number of times the j-th datum \\((x_j, y_j)\\) is selected in the b-th bootstrap dataset.</p> \\[\\psi((x_i,y_i),\\Theta_B):=\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i, \\hat{f}_b(x_i))}{\\sum_{b=1}^{B} \\mathbb{1} (w_{bi}=0)}\\] <p>With:</p> \\[ T: Y \\times Y \\rightarrow \\mathbb{R} \\] <p>T is a score function that represents the goodness of a weak learner \\(\\hat{f}_b\\) at the i-th datum \\((x_i, y_i)\\).</p> <p><code>n_est</code> and <code>max_samples</code> must be tuned jointly to ensure that all samples are at least 1 time out-of-bag, otherwise the result could include a NaN value for that datum.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>n_est</code> <p>Number of estimator used in the bagging procedure.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>max_samples</code> <p>The fraction of samples to draw to train each base estimator.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>loss</code> <p>A function taking as parameters model prediction and corresponding data labels(y_true, y_pred) and returning an array of point-wise errors.</p> <p> TYPE: <code>Optional[LossFunction]</code> DEFAULT: <code>None</code> </p> <code>n_jobs</code> <p>The number of jobs to run in parallel used in the bagging procedure for both fit and predict.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def compute_data_oob(\n    u: Utility,\n    *,\n    n_est: int = 10,\n    max_samples: float = 0.8,\n    loss: Optional[LossFunction] = None,\n    n_jobs: Optional[int] = None,\n    seed: Optional[Seed] = None,\n    progress: bool = False,\n) -&gt; ValuationResult:\n    r\"\"\"Computes Data out of bag values\n\n    This implements the method described in\n    (Kwon and Zou, 2023)&lt;sup&gt;&lt;a href=\"kwon_data_2023\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    It fits several base estimators provided through u.model through a bagging\n    process. The point value corresponds to the average loss of estimators which\n    were not fit on it.\n\n    $w_{bj}\\in Z$ is the number of times the j-th datum $(x_j, y_j)$ is selected\n    in the b-th bootstrap dataset.\n\n    $$\\psi((x_i,y_i),\\Theta_B):=\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i,\n    \\hat{f}_b(x_i))}{\\sum_{b=1}^{B}\n    \\mathbb{1}\n    (w_{bi}=0)}$$\n\n    With:\n\n    $$\n    T: Y \\times Y\n    \\rightarrow \\mathbb{R}\n    $$\n\n    T is a score function that represents the goodness of a weak learner\n    $\\hat{f}_b$ at the i-th datum $(x_i, y_i)$.\n\n    `n_est` and `max_samples` must be tuned jointly to ensure that all samples\n    are at least 1 time out-of-bag, otherwise the result could include a NaN\n    value for that datum.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        n_est: Number of estimator used in the bagging procedure.\n        max_samples: The fraction of samples to draw to train each base\n            estimator.\n        loss: A function taking as parameters model prediction and corresponding\n            data labels(y_true, y_pred) and returning an array of point-wise errors.\n        n_jobs: The number of jobs to run in parallel used in the bagging\n            procedure for both fit and predict.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        progress: If True, display a progress bar.\n\n    Returns:\n        Object with the data values.\n    \"\"\"\n    rng = np.random.default_rng(seed)\n    random_state = np.random.RandomState(rng.bit_generator)\n\n    result: ValuationResult[np.int_, np.object_] = ValuationResult.empty(\n        algorithm=\"data_oob\", indices=u.data.indices, data_names=u.data.data_names\n    )\n\n    if is_classifier(u.model):\n        bag = BaggingClassifier(\n            u.model,\n            n_estimators=n_est,\n            max_samples=max_samples,\n            n_jobs=n_jobs,\n            random_state=random_state,\n        )\n        if loss is None:\n            loss = point_wise_accuracy\n    elif is_regressor(u.model):\n        bag = BaggingRegressor(\n            u.model,\n            n_estimators=n_est,\n            max_samples=max_samples,\n            n_jobs=n_jobs,\n            random_state=random_state,\n        )\n        if loss is None:\n            loss = neg_l2_distance\n    else:\n        raise Exception(\n            \"Model has to be a classifier or a regressor in sklearn format.\"\n        )\n\n    bag.fit(u.data.x_train, u.data.y_train)\n\n    for est, samples in tqdm(\n        zip(bag.estimators_, bag.estimators_samples_), disable=not progress, total=n_est\n    ):  # The bottleneck is the bag fitting not this part so TQDM is not very useful here\n        oob_idx = np.setxor1d(u.data.indices, np.unique(samples))\n        array_loss = loss(\n            y_true=u.data.y_train[oob_idx],\n            y_pred=est.predict(u.data.x_train[oob_idx]),\n        )\n        result += ValuationResult(\n            algorithm=\"data_oob\",\n            indices=oob_idx,\n            values=array_loss,\n            counts=np.ones_like(array_loss, dtype=u.data.indices.dtype),\n        )\n    return result\n</code></pre>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.point_wise_accuracy","title":"point_wise_accuracy","text":"<pre><code>point_wise_accuracy(y_true: NDArray[T], y_pred: NDArray[T]) -&gt; NDArray[T]\n</code></pre> <p>Point-wise 0-1 loss between two arrays</p> PARAMETER  DESCRIPTION <code>y_true</code> <p>Array of true values (e.g. labels)</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>y_pred</code> <p>Array of estimated values (e.g. model predictions)</p> <p> TYPE: <code>NDArray[T]</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>Array with point-wise 0-1 losses between labels and model predictions</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def point_wise_accuracy(y_true: NDArray[T], y_pred: NDArray[T]) -&gt; NDArray[T]:\n    r\"\"\"Point-wise 0-1 loss between two arrays\n\n    Args:\n        y_true: Array of true values (e.g. labels)\n        y_pred: Array of estimated values (e.g. model predictions)\n\n    Returns:\n        Array with point-wise 0-1 losses between labels and model predictions\n    \"\"\"\n    return np.array(y_pred == y_true, dtype=y_pred.dtype)\n</code></pre>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.neg_l2_distance","title":"neg_l2_distance","text":"<pre><code>neg_l2_distance(y_true: NDArray[T], y_pred: NDArray[T]) -&gt; NDArray[T]\n</code></pre> <p>Point-wise negative \\(l_2\\) distance between two arrays</p> PARAMETER  DESCRIPTION <code>y_true</code> <p>Array of true values (e.g. labels)</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>y_pred</code> <p>Array of estimated values (e.g. model predictions)</p> <p> TYPE: <code>NDArray[T]</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>Array with point-wise negative \\(l_2\\) distances between labels and model</p> <code>NDArray[T]</code> <p>predictions</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def neg_l2_distance(y_true: NDArray[T], y_pred: NDArray[T]) -&gt; NDArray[T]:\n    r\"\"\"Point-wise negative $l_2$ distance between two arrays\n\n    Args:\n        y_true: Array of true values (e.g. labels)\n        y_pred: Array of estimated values (e.g. model predictions)\n\n    Returns:\n        Array with point-wise negative $l_2$ distances between labels and model\n        predictions\n    \"\"\"\n    return -np.square(np.array(y_pred - y_true), dtype=y_pred.dtype)\n</code></pre>"},{"location":"api/pydvl/value/shapley/","title":"Shapley","text":""},{"location":"api/pydvl/value/shapley/#pydvl.value.shapley","title":"pydvl.value.shapley","text":"<p>This package holds all routines for the computation of Shapley Data value. Users will want to use compute_shapley_values or compute_semivalues as interfaces to most methods defined in the modules.</p> <p>Please refer to the guide on data valuation for an overview of all methods.</p>"},{"location":"api/pydvl/value/shapley/classwise/","title":"Classwise","text":""},{"location":"api/pydvl/value/shapley/classwise/#pydvl.value.shapley.classwise","title":"pydvl.value.shapley.classwise","text":"<p>Class-wise Shapley (Schoch et al., 2022)<sup>1</sup> offers a Shapley framework tailored for classification problems. Let \\(D\\) be a dataset, \\(D_{y_i}\\) be the subset of \\(D\\) with labels \\(y_i\\), and \\(D_{-y_i}\\) be the complement of \\(D_{y_i}\\) in \\(D\\). The key idea is that a sample \\((x_i, y_i)\\), might enhance the overall performance on \\(D\\), while being detrimental for the performance on \\(D_{y_i}\\). The Class-wise value is defined as:</p> \\[ v_u(i) = \\frac{1}{2^{|D_{-y_i}|}} \\sum_{S_{-y_i}} \\frac{1}{|D_{y_i}|!} \\sum_{S_{y_i}} \\binom{|D_{y_i}|-1}{|S_{y_i}|}^{-1} [u( S_{y_i} \\cup \\{i\\} | S_{-y_i} ) \u2212 u( S_{y_i} | S_{-y_i})], \\] <p>where \\(S_{y_i} \\subseteq D_{y_i} \\setminus \\{i\\}\\) and \\(S_{-y_i} \\subseteq D_{-y_i}\\).</p> <p>Analysis of Class-wise Shapley</p> <p>For a detailed analysis of the method, with comparison to other valuation techniques, please refer to the main documentation.</p> <p>In practice, the quantity above is estimated using Monte Carlo sampling of the powerset and the set of index permutations. This results in the estimator</p> \\[ v_u(i) = \\frac{1}{K} \\sum_k \\frac{1}{L} \\sum_l [u(\\sigma^{(l)}_{:i} \\cup \\{i\\} | S^{(k)} ) \u2212 u( \\sigma^{(l)}_{:i} | S^{(k)})], \\] <p>with \\(S^{(1)}, \\dots, S^{(K)} \\subseteq T_{-y_i},\\) \\(\\sigma^{(1)}, \\dots, \\sigma^{(L)} \\in \\Pi(T_{y_i}\\setminus\\{i\\}),\\) and \\(\\sigma^{(l)}_{:i}\\) denoting the set of indices in permutation \\(\\sigma^{(l)}\\) before the position where \\(i\\) appears. The sets \\(T_{y_i}\\) and \\(T_{-y_i}\\) are the training sets for the labels \\(y_i\\) and \\(-y_i\\), respectively.</p> Notes for derivation of test cases <p>The unit tests include the following manually constructed data: Let \\(D=\\{(1,0),(2,0),(3,0),(4,1)\\}\\) be the test set and \\(T=\\{(1,0),(2,0),(3,1),(4,1)\\}\\) the train set. This specific dataset is chosen as it allows to solve the model</p> \\[y = \\max(0, \\min(1, \\text{round}(\\beta^T x)))\\] <p>in closed form \\(\\beta = \\frac{\\text{dot}(x, y)}{\\text{dot}(x, x)}\\). From the closed-form solution, the tables for in-class accuracy \\(a_S(D_{y_i})\\) and out-of-class accuracy  \\(a_S(D_{-y_i})\\) can be calculated. By using these tables and setting  \\(\\{S^{(1)}, \\dots, S^{(K)}\\} = 2^{T_{-y_i}}\\) and  \\(\\{\\sigma^{(1)}, \\dots, \\sigma^{(L)}\\} = \\Pi(T_{y_i}\\setminus\\{i\\})\\), the Monte Carlo estimator can be evaluated (\\(2^M\\) is the powerset of \\(M\\)). The details of the derivation are left to the eager reader.</p>"},{"location":"api/pydvl/value/shapley/classwise/#pydvl.value.shapley.classwise--references","title":"References","text":"<ol> <li> <p>Schoch, Stephanie, Haifeng Xu, and Yangfeng Ji. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification. In Proc. of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). New Orleans, Louisiana, USA, 2022.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/classwise/#pydvl.value.shapley.classwise.ClasswiseScorer","title":"ClasswiseScorer","text":"<pre><code>ClasswiseScorer(\n    scoring: Union[str, ScorerCallable] = \"accuracy\",\n    default: float = 0.0,\n    range: Tuple[float, float] = (0, 1),\n    in_class_discount_fn: Callable[[float], float] = lambda x: x,\n    out_of_class_discount_fn: Callable[[float], float] = np.exp,\n    initial_label: Optional[int] = None,\n    name: Optional[str] = None,\n)\n</code></pre> <p>             Bases: <code>Scorer</code></p> <p>A Scorer designed for evaluation in classification problems. Its value is computed from an in-class and an out-of-class \"inner score\" (Schoch et al., 2022) <sup>1</sup>. Let \\(S\\) be the training set and \\(D\\) be the valuation set. For each label \\(c\\), \\(D\\) is factorized into two disjoint sets: \\(D_c\\) for in-class instances and \\(D_{-c}\\) for out-of-class instances. The score combines an in-class metric of performance, adjusted by a discounted out-of-class metric. These inner scores must be provided upon construction or default to accuracy. They are combined into:</p> \\[ u(S_{y_i}) = f(a_S(D_{y_i}))\\ g(a_S(D_{-y_i})), \\] <p>where \\(f\\) and \\(g\\) are continuous, monotonic functions. For a detailed explanation, refer to section four of (Schoch et al., 2022)<sup> 1</sup>.</p> <p>Warning</p> <p>Metrics must support multiple class labels if you intend to apply them to a multi-class problem. For instance, the metric 'accuracy' supports multiple classes, but the metric <code>f1</code> does not. For a two-class classification problem, using <code>f1_weighted</code> is essentially equivalent to using <code>accuracy</code>.</p> PARAMETER  DESCRIPTION <code>scoring</code> <p>Name of the scoring function or a callable that can be passed to Scorer.</p> <p> TYPE: <code>Union[str, ScorerCallable]</code> DEFAULT: <code>'accuracy'</code> </p> <code>default</code> <p>Score to use when a model fails to provide a number, e.g. when too little was used to train it, or errors arise.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>range</code> <p>Numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation. If not provided, it can be read from the <code>scoring</code> object if it provides it, for instance if it was constructed with compose_score.</p> <p> TYPE: <code>Tuple[float, float]</code> DEFAULT: <code>(0, 1)</code> </p> <code>in_class_discount_fn</code> <p>Continuous, monotonic increasing function used to discount the in-class score.</p> <p> TYPE: <code>Callable[[float], float]</code> DEFAULT: <code>lambda x: x</code> </p> <code>out_of_class_discount_fn</code> <p>Continuous, monotonic increasing function used to discount the out-of-class score.</p> <p> TYPE: <code>Callable[[float], float]</code> DEFAULT: <code>exp</code> </p> <code>initial_label</code> <p>Set initial label (for the first iteration)</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>name</code> <p>Name of the scorer. If not provided, the name of the inner scoring function will be prefixed by <code>classwise</code>.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <p>New in version 0.7.1</p> Source code in <code>src/pydvl/value/shapley/classwise.py</code> <pre><code>def __init__(\n    self,\n    scoring: Union[str, ScorerCallable] = \"accuracy\",\n    default: float = 0.0,\n    range: Tuple[float, float] = (0, 1),\n    in_class_discount_fn: Callable[[float], float] = lambda x: x,\n    out_of_class_discount_fn: Callable[[float], float] = np.exp,\n    initial_label: Optional[int] = None,\n    name: Optional[str] = None,\n):\n    disc_score_in_class = in_class_discount_fn(range[1])\n    disc_score_out_of_class = out_of_class_discount_fn(range[1])\n    transformed_range = (0, disc_score_in_class * disc_score_out_of_class)\n    super().__init__(\n        scoring=scoring,\n        range=transformed_range,\n        default=default,\n        name=name or f\"classwise {str(scoring)}\",\n    )\n    self._in_class_discount_fn = in_class_discount_fn\n    self._out_of_class_discount_fn = out_of_class_discount_fn\n    self.label = initial_label\n</code></pre>"},{"location":"api/pydvl/value/shapley/classwise/#pydvl.value.shapley.classwise.ClasswiseScorer.estimate_in_class_and_out_of_class_score","title":"estimate_in_class_and_out_of_class_score","text":"<pre><code>estimate_in_class_and_out_of_class_score(\n    model: SupervisedModel,\n    x_test: NDArray[float_],\n    y_test: NDArray[int_],\n    rescale_scores: bool = True,\n) -&gt; Tuple[float, float]\n</code></pre> <p>Computes in-class and out-of-class scores using the provided inner scoring function. The result is</p> \\[ a_S(D=\\{(x_1, y_1), \\dots, (x_K, y_K)\\}) = \\frac{1}{N} \\sum_k s(y(x_k), y_k). \\] <p>In this context, for label \\(c\\) calculations are executed twice: once for \\(D_c\\) and once for \\(D_{-c}\\) to determine the in-class and out-of-class scores, respectively. By default, the raw scores are multiplied by \\(\\frac{|D_c|}{|D|}\\) and \\(\\frac{|D_{-c}|}{|D|}\\), respectively. This is done to ensure that both scores are of the same order of magnitude. This normalization is particularly useful when the inner score function \\(a_S\\) is calculated by an estimator of the form \\(\\frac{1}{N} \\sum_i x_i\\), e.g. the accuracy.</p> PARAMETER  DESCRIPTION <code>model</code> <p>Model used for computing the score on the validation set.</p> <p> TYPE: <code>SupervisedModel</code> </p> <code>x_test</code> <p>Array containing the features of the classification problem.</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>y_test</code> <p>Array containing the labels of the classification problem.</p> <p> TYPE: <code>NDArray[int_]</code> </p> <code>rescale_scores</code> <p>If set to True, the scores will be denormalized. This is particularly useful when the inner score function \\(a_S\\) is calculated by an estimator of the form \\(\\frac{1}{N} \\sum_i x_i\\).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>Tuple[float, float]</code> <p>Tuple containing the in-class and out-of-class scores.</p> Source code in <code>src/pydvl/value/shapley/classwise.py</code> <pre><code>def estimate_in_class_and_out_of_class_score(\n    self,\n    model: SupervisedModel,\n    x_test: NDArray[np.float_],\n    y_test: NDArray[np.int_],\n    rescale_scores: bool = True,\n) -&gt; Tuple[float, float]:\n    r\"\"\"\n    Computes in-class and out-of-class scores using the provided inner\n    scoring function. The result is\n\n    $$\n    a_S(D=\\{(x_1, y_1), \\dots, (x_K, y_K)\\}) = \\frac{1}{N} \\sum_k s(y(x_k), y_k).\n    $$\n\n    In this context, for label $c$ calculations are executed twice: once for $D_c$\n    and once for $D_{-c}$ to determine the in-class and out-of-class scores,\n    respectively. By default, the raw scores are multiplied by $\\frac{|D_c|}{|D|}$\n    and $\\frac{|D_{-c}|}{|D|}$, respectively. This is done to ensure that both\n    scores are of the same order of magnitude. This normalization is particularly\n    useful when the inner score function $a_S$ is calculated by an estimator of the\n    form $\\frac{1}{N} \\sum_i x_i$, e.g. the accuracy.\n\n    Args:\n        model: Model used for computing the score on the validation set.\n        x_test: Array containing the features of the classification problem.\n        y_test: Array containing the labels of the classification problem.\n        rescale_scores: If set to True, the scores will be denormalized. This is\n            particularly useful when the inner score function $a_S$ is calculated by\n            an estimator of the form $\\frac{1}{N} \\sum_i x_i$.\n\n    Returns:\n        Tuple containing the in-class and out-of-class scores.\n    \"\"\"\n    scorer = self._scorer\n    label_set_match = y_test == self.label\n    label_set = np.where(label_set_match)[0]\n    num_classes = len(np.unique(y_test))\n\n    if len(label_set) == 0:\n        return 0, 1 / (num_classes - 1)\n\n    complement_label_set = np.where(~label_set_match)[0]\n    in_class_score = scorer(model, x_test[label_set], y_test[label_set])\n    out_of_class_score = scorer(\n        model, x_test[complement_label_set], y_test[complement_label_set]\n    )\n\n    if rescale_scores:\n        n_in_class = np.count_nonzero(y_test == self.label)\n        n_out_of_class = len(y_test) - n_in_class\n        in_class_score *= n_in_class / (n_in_class + n_out_of_class)\n        out_of_class_score *= n_out_of_class / (n_in_class + n_out_of_class)\n\n    return in_class_score, out_of_class_score\n</code></pre>"},{"location":"api/pydvl/value/shapley/classwise/#pydvl.value.shapley.classwise.compute_classwise_shapley_values","title":"compute_classwise_shapley_values","text":"<pre><code>compute_classwise_shapley_values(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    truncation: TruncationPolicy,\n    done_sample_complements: Optional[StoppingCriterion] = None,\n    normalize_values: bool = True,\n    use_default_scorer_value: bool = True,\n    min_elements_per_label: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes an approximate Class-wise Shapley value by sampling independent permutations of the index set for each label and index sets sampled from the powerset of the complement (with respect to the currently evaluated label), approximating the sum:</p> \\[ v_u(i) = \\frac{1}{K} \\sum_k \\frac{1}{L} \\sum_l [u(\\sigma^{(l)}_{:i} \\cup \\{i\\} | S^{(k)} ) \u2212 u( \\sigma^{(l)}_{:i} | S^{(k)})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears and \\(S\\) is a subset of the index set of all other labels (see the main documentation for details).</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object containing model, data, and scoring function. The scorer must be of type ClasswiseScorer.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Function that checks whether the computation needs to stop.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>truncation</code> <p>Callable function that decides whether to interrupt processing a permutation and set subsequent marginals to zero.</p> <p> TYPE: <code>TruncationPolicy</code> </p> <code>done_sample_complements</code> <p>Function checking whether computation needs to stop. Otherwise, it will resample conditional sets until the stopping criterion is met.</p> <p> TYPE: <code>Optional[StoppingCriterion]</code> DEFAULT: <code>None</code> </p> <code>normalize_values</code> <p>Indicates whether to normalize the values by the variation in each class times their in-class accuracy.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>done_sample_complements</code> <p>Number of times to resample the complement set for each permutation.</p> <p> TYPE: <code>Optional[StoppingCriterion]</code> DEFAULT: <code>None</code> </p> <code>use_default_scorer_value</code> <p>The first set of indices is the sampled complement set. Unless not otherwise specified, the default scorer value is used for this. If it is set to false, the base score is calculated from the utility.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>min_elements_per_label</code> <p>The minimum number of elements for each opposite label.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to run.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>ValuationResult object containing computed data values.</p> <p>New in version 0.7.1</p> Source code in <code>src/pydvl/value/shapley/classwise.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_classwise_shapley_values(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    truncation: TruncationPolicy,\n    done_sample_complements: Optional[StoppingCriterion] = None,\n    normalize_values: bool = True,\n    use_default_scorer_value: bool = True,\n    min_elements_per_label: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    r\"\"\"\n    Computes an approximate Class-wise Shapley value by sampling independent\n    permutations of the index set for each label and index sets sampled from the\n    powerset of the complement (with respect to the currently evaluated label),\n    approximating the sum:\n\n    $$\n    v_u(i) = \\frac{1}{K} \\sum_k \\frac{1}{L} \\sum_l\n    [u(\\sigma^{(l)}_{:i} \\cup \\{i\\} | S^{(k)} ) \u2212 u( \\sigma^{(l)}_{:i} | S^{(k)})],\n    $$\n\n    where $\\sigma_{:i}$ denotes the set of indices in permutation sigma before\n    the position where $i$ appears and $S$ is a subset of the index set of all\n    other labels (see [the main documentation][class-wise-shapley] for\n    details).\n\n    Args:\n        u: Utility object containing model, data, and scoring function. The\n            scorer must be of type\n            [ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer].\n        done: Function that checks whether the computation needs to stop.\n        truncation: Callable function that decides whether to interrupt processing a\n            permutation and set subsequent marginals to zero.\n        done_sample_complements: Function checking whether computation needs to stop.\n            Otherwise, it will resample conditional sets until the stopping criterion is\n            met.\n        normalize_values: Indicates whether to normalize the values by the variation\n            in each class times their in-class accuracy.\n        done_sample_complements: Number of times to resample the complement set\n            for each permutation.\n        use_default_scorer_value: The first set of indices is the sampled complement\n            set. Unless not otherwise specified, the default scorer value is used for\n            this. If it is set to false, the base score is calculated from the utility.\n        min_elements_per_label: The minimum number of elements for each opposite\n            label.\n        n_jobs: Number of parallel jobs to run.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        ValuationResult object containing computed data values.\n\n    !!! tip \"New in version 0.7.1\"\n    \"\"\"\n    dim_correct = u.data.y_train.ndim == 1 and u.data.y_test.ndim == 1\n    is_integral = all(\n        map(\n            lambda v: isinstance(v, numbers.Integral), (*u.data.y_train, *u.data.y_test)\n        )\n    )\n    if not dim_correct or not is_integral:\n        raise ValueError(\n            \"The supplied dataset has to be a 1-dimensional classification dataset.\"\n        )\n\n    if not isinstance(u.scorer, ClasswiseScorer):\n        raise ValueError(\n            \"Please set a subclass of ClasswiseScorer object as scorer object of the\"\n            \" utility. See scoring argument of Utility.\"\n        )\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n    u_ref = parallel_backend.put(u)\n    n_jobs = parallel_backend.effective_n_jobs(n_jobs)\n    n_submitted_jobs = 2 * n_jobs\n\n    pbar = tqdm(disable=not progress, position=0, total=100, unit=\"%\")\n    algorithm = \"classwise_shapley\"\n    accumulated_result = ValuationResult.zeros(\n        algorithm=algorithm, indices=u.data.indices, data_names=u.data.data_names\n    )\n    terminate_exec = False\n    seed_sequence = ensure_seed_sequence(seed)\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    with parallel_backend.executor(max_workers=n_jobs) as executor:\n        pending: Set[Future] = set()\n        while True:\n            completed_futures, pending = wait(\n                pending, timeout=60, return_when=FIRST_COMPLETED\n            )\n            for future in completed_futures:\n                accumulated_result += future.result()\n                if done(accumulated_result):\n                    terminate_exec = True\n                    break\n\n            pbar.n = 100 * done.completion()\n            pbar.refresh()\n            if terminate_exec:\n                break\n\n            n_remaining_slots = n_submitted_jobs - len(pending)\n            seeds = seed_sequence.spawn(n_remaining_slots)\n            for i in range(n_remaining_slots):\n                future = executor.submit(\n                    _permutation_montecarlo_classwise_shapley_one_step,\n                    u_ref,\n                    truncation=truncation,\n                    done_sample_complements=done_sample_complements,\n                    use_default_scorer_value=use_default_scorer_value,\n                    min_elements_per_label=min_elements_per_label,\n                    algorithm_name=algorithm,\n                    seed=seeds[i],\n                )\n                pending.add(future)\n\n    result = accumulated_result\n    if normalize_values:\n        result = _normalize_classwise_shapley_values(result, u)\n\n    return result\n</code></pre>"},{"location":"api/pydvl/value/shapley/common/","title":"Common","text":""},{"location":"api/pydvl/value/shapley/common/#pydvl.value.shapley.common","title":"pydvl.value.shapley.common","text":""},{"location":"api/pydvl/value/shapley/common/#pydvl.value.shapley.common.compute_shapley_values","title":"compute_shapley_values","text":"<pre><code>compute_shapley_values(\n    u: Utility,\n    *,\n    done: StoppingCriterion = MaxChecks(None),\n    mode: ShapleyMode = ShapleyMode.TruncatedMontecarlo,\n    n_jobs: int = 1,\n    seed: Optional[Seed] = None,\n    **kwargs\n) -&gt; ValuationResult\n</code></pre> <p>Umbrella method to compute Shapley values with any of the available algorithms.</p> <p>See Data valuation for an overview.</p> <p>The following algorithms are available. Note that the exact methods can only work with very small datasets and are thus intended only for testing. Some algorithms also accept additional arguments, please refer to the documentation of each particular method.</p> <ul> <li><code>combinatorial_exact</code>: uses the combinatorial implementation of data   Shapley. Implemented in   combinatorial_exact_shapley().</li> <li><code>combinatorial_montecarlo</code>:  uses the approximate Monte Carlo   implementation of combinatorial data Shapley. Implemented in   combinatorial_montecarlo_shapley().</li> <li><code>permutation_exact</code>: uses the permutation-based implementation of data   Shapley. Computation is not parallelized. Implemented in   permutation_exact_shapley().</li> <li><code>permutation_montecarlo</code>: uses the approximate Monte Carlo   implementation of permutation data Shapley. Accepts a   TruncationPolicy to stop   computing marginals. Implemented in   permutation_montecarlo_shapley().</li> <li><code>owen_sampling</code>: Uses the Owen continuous extension of the utility   function to the unit cube. Implemented in   owen_sampling_shapley(). This   method does not take a StoppingCriterion   but instead requires a parameter <code>q_max</code> for the number of subdivisions   of the unit interval to use for integration, and another parameter   <code>n_samples</code> for the number of subsets to sample for each \\(q\\).</li> <li><code>owen_halved</code>: Same as 'owen_sampling' but uses correlated samples in the   expectation. Implemented in   owen_sampling_shapley().   This method  requires an additional parameter <code>q_max</code> for the number of   subdivisions of the interval [0,0.5] to use for integration, and another   parameter <code>n_samples</code> for the number of subsets to sample for each \\(q\\).</li> <li><code>group_testing</code>: estimates differences of Shapley values and solves a   constraint satisfaction problem. High sample complexity, not recommended.   Implemented in group_testing_shapley(). This   method does not take a StoppingCriterion   but instead requires a parameter <code>n_samples</code> for the number of   iterations to run.</li> </ul> <p>Additionally, one can use model-specific methods:</p> <ul> <li><code>knn</code>: Exact method for K-Nearest neighbour models. Implemented in   knn_shapley().</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Object used to determine when to stop the computation for Monte Carlo methods. The default is to stop after 100 iterations. See the available criteria in stopping. It is possible to combine several of them using boolean operators. Some methods ignore this argument, others require specific subtypes.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxChecks(None)</code> </p> <code>n_jobs</code> <p>Number of parallel jobs (available only to some methods)</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>Choose which shapley algorithm to use. See ShapleyMode for a list of allowed value.</p> <p> TYPE: <code>ShapleyMode</code> DEFAULT: <code>TruncatedMontecarlo</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/shapley/common.py</code> <pre><code>def compute_shapley_values(\n    u: Utility,\n    *,\n    done: StoppingCriterion = MaxChecks(None),\n    mode: ShapleyMode = ShapleyMode.TruncatedMontecarlo,\n    n_jobs: int = 1,\n    seed: Optional[Seed] = None,\n    **kwargs,\n) -&gt; ValuationResult:\n    \"\"\"Umbrella method to compute Shapley values with any of the available\n    algorithms.\n\n    See [Data valuation][data-valuation] for an overview.\n\n    The following algorithms are available. Note that the exact methods can only\n    work with very small datasets and are thus intended only for testing. Some\n    algorithms also accept additional arguments, please refer to the\n    documentation of each particular method.\n\n    - `combinatorial_exact`: uses the combinatorial implementation of data\n      Shapley. Implemented in\n      [combinatorial_exact_shapley()][pydvl.value.shapley.naive.combinatorial_exact_shapley].\n    - `combinatorial_montecarlo`:  uses the approximate Monte Carlo\n      implementation of combinatorial data Shapley. Implemented in\n      [combinatorial_montecarlo_shapley()][pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley].\n    - `permutation_exact`: uses the permutation-based implementation of data\n      Shapley. Computation is **not parallelized**. Implemented in\n      [permutation_exact_shapley()][pydvl.value.shapley.naive.permutation_exact_shapley].\n    - `permutation_montecarlo`: uses the approximate Monte Carlo\n      implementation of permutation data Shapley. Accepts a\n      [TruncationPolicy][pydvl.value.shapley.truncated.TruncationPolicy] to stop\n      computing marginals. Implemented in\n      [permutation_montecarlo_shapley()][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].\n    - `owen_sampling`: Uses the Owen continuous extension of the utility\n      function to the unit cube. Implemented in\n      [owen_sampling_shapley()][pydvl.value.shapley.owen.owen_sampling_shapley]. This\n      method does not take a [StoppingCriterion][pydvl.value.stopping.StoppingCriterion]\n      but instead requires a parameter `q_max` for the number of subdivisions\n      of the unit interval to use for integration, and another parameter\n      `n_samples` for the number of subsets to sample for each $q$.\n    - `owen_halved`: Same as 'owen_sampling' but uses correlated samples in the\n      expectation. Implemented in\n      [owen_sampling_shapley()][pydvl.value.shapley.owen.owen_sampling_shapley].\n      This method  requires an additional parameter `q_max` for the number of\n      subdivisions of the interval [0,0.5] to use for integration, and another\n      parameter `n_samples` for the number of subsets to sample for each $q$.\n    - `group_testing`: estimates differences of Shapley values and solves a\n      constraint satisfaction problem. High sample complexity, not recommended.\n      Implemented in [group_testing_shapley()][pydvl.value.shapley.gt.group_testing_shapley]. This\n      method does not take a [StoppingCriterion][pydvl.value.stopping.StoppingCriterion]\n      but instead requires a parameter `n_samples` for the number of\n      iterations to run.\n\n    Additionally, one can use model-specific methods:\n\n    - `knn`: Exact method for K-Nearest neighbour models. Implemented in\n      [knn_shapley()][pydvl.value.shapley.knn.knn_shapley].\n\n    Args:\n        u: [Utility][pydvl.utils.utility.Utility] object with model, data, and\n            scoring function.\n        done: Object used to determine when to stop the computation for Monte\n            Carlo methods. The default is to stop after 100 iterations. See the\n            available criteria in [stopping][pydvl.value.stopping]. It is\n            possible to combine several of them using boolean operators. Some\n            methods ignore this argument, others require specific subtypes.\n        n_jobs: Number of parallel jobs (available only to some methods)\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        mode: Choose which shapley algorithm to use. See\n            [ShapleyMode][pydvl.value.shapley.ShapleyMode] for a list of allowed\n            value.\n\n    Returns:\n        Object with the results.\n\n    \"\"\"\n    progress: bool = kwargs.pop(\"progress\", False)\n\n    if mode not in list(ShapleyMode):\n        raise ValueError(f\"Invalid value encountered in {mode=}\")\n\n    if mode in (\n        ShapleyMode.PermutationMontecarlo,\n        ShapleyMode.ApproShapley,\n        ShapleyMode.TruncatedMontecarlo,\n    ):\n        truncation = kwargs.pop(\"truncation\", NoTruncation())\n        return permutation_montecarlo_shapley(  # type: ignore\n            u=u,\n            done=done,\n            truncation=truncation,\n            n_jobs=n_jobs,\n            seed=seed,\n            progress=progress,\n            **kwargs,\n        )\n    elif mode == ShapleyMode.CombinatorialMontecarlo:\n        return combinatorial_montecarlo_shapley(  # type: ignore\n            u, done=done, n_jobs=n_jobs, seed=seed, progress=progress\n        )\n    elif mode == ShapleyMode.CombinatorialExact:\n        return combinatorial_exact_shapley(u, n_jobs=n_jobs, progress=progress)  # type: ignore\n    elif mode == ShapleyMode.PermutationExact:\n        return permutation_exact_shapley(u, progress=progress)\n    elif mode == ShapleyMode.Owen or mode == ShapleyMode.OwenAntithetic:\n        if kwargs.get(\"n_samples\") is None:\n            raise ValueError(\"n_samples cannot be None for Owen methods\")\n        if kwargs.get(\"max_q\") is None:\n            raise ValueError(\"Owen Sampling requires max_q for the outer integral\")\n\n        method = (\n            OwenAlgorithm.Standard\n            if mode == ShapleyMode.Owen\n            else OwenAlgorithm.Antithetic\n        )\n        return owen_sampling_shapley(  # type: ignore\n            u,\n            n_samples=int(kwargs.get(\"n_samples\", -1)),\n            max_q=int(kwargs.get(\"max_q\", -1)),\n            method=method,\n            n_jobs=n_jobs,\n            seed=seed,\n        )\n    elif mode == ShapleyMode.KNN:\n        return knn_shapley(u, progress=progress)\n    elif mode == ShapleyMode.GroupTesting:\n        n_samples = kwargs.pop(\"n_samples\")\n        if n_samples is None:\n            raise ValueError(\"n_samples cannot be None for Group Testing\")\n        epsilon = kwargs.pop(\"epsilon\")\n        if epsilon is None:\n            raise ValueError(\"Group Testing requires error bound epsilon\")\n        delta = kwargs.pop(\"delta\", 0.05)\n        return group_testing_shapley(  # type: ignore\n            u,\n            epsilon=float(epsilon),\n            delta=delta,\n            n_samples=int(n_samples),\n            n_jobs=n_jobs,\n            progress=progress,\n            seed=seed,\n            **kwargs,\n        )\n    else:\n        raise ValueError(f\"Invalid value encountered in {mode=}\")\n</code></pre>"},{"location":"api/pydvl/value/shapley/gt/","title":"Gt","text":""},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt","title":"pydvl.value.shapley.gt","text":"<p>This module implements Group Testing for the approximation of Shapley values, as introduced in (Jia, R. et al., 2019)<sup>1</sup>. The sampling of index subsets is done in such a way that an approximation to the true Shapley values can be computed with guarantees.</p> <p>Warning</p> <p>This method is very inefficient. Potential improvements to the implementation notwithstanding, convergence seems to be very slow (in terms of evaluations of the utility required). We recommend other Monte Carlo methods instead.</p> <p>You can read more in the documentation.</p> <p>New in version 0.4.0</p>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt--references","title":"References","text":"<ol> <li> <p>Jia, R. et al., 2019. Towards Efficient Data Valuation Based on the Shapley Value. In: Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics, pp. 1167\u20131176. PMLR.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt.num_samples_eps_delta","title":"num_samples_eps_delta","text":"<pre><code>num_samples_eps_delta(\n    eps: float, delta: float, n: int, utility_range: float\n) -&gt; int\n</code></pre> <p>Implements the formula in Theorem 3 of (Jia, R. et al., 2019)<sup>1</sup> which gives a lower bound on the number of samples required to obtain an (\u03b5/\u221an,\u03b4/(N(N-1))-approximation to all pair-wise differences of Shapley values, wrt. \\(\\ell_2\\) norm.</p> PARAMETER  DESCRIPTION <code>eps</code> <p>\u03b5</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>\u03b4</p> <p> TYPE: <code>float</code> </p> <code>n</code> <p>Number of data points</p> <p> TYPE: <code>int</code> </p> <code>utility_range</code> <p>Range of the Utility function</p> <p> TYPE: <code>float</code> </p> <p>Returns:     Number of samples from \\(2^{[n]}\\) guaranteeing \u03b5/\u221an-correct Shapley         pair-wise differences of values with probability 1-\u03b4/(N(N-1)).</p> <p>New in version 0.4.0</p> Source code in <code>src/pydvl/value/shapley/gt.py</code> <pre><code>def num_samples_eps_delta(\n    eps: float, delta: float, n: int, utility_range: float\n) -&gt; int:\n    r\"\"\"Implements the formula in Theorem 3 of (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n    which gives a lower bound on the number of samples required to obtain an\n    (\u03b5/\u221an,\u03b4/(N(N-1))-approximation to all pair-wise differences of Shapley\n    values, wrt. $\\ell_2$ norm.\n\n    Args:\n        eps: \u03b5\n        delta: \u03b4\n        n: Number of data points\n        utility_range: Range of the [Utility][pydvl.utils.utility.Utility] function\n    Returns:\n        Number of samples from $2^{[n]}$ guaranteeing \u03b5/\u221an-correct Shapley\n            pair-wise differences of values with probability 1-\u03b4/(N(N-1)).\n\n    !!! tip \"New in version 0.4.0\"\n\n    \"\"\"\n    constants = _constants(n=n, epsilon=eps, delta=delta, utility_range=utility_range)\n    return int(constants.T)\n</code></pre>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt.group_testing_shapley","title":"group_testing_shapley","text":"<pre><code>group_testing_shapley(\n    u: Utility,\n    n_samples: int,\n    epsilon: float,\n    delta: float,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n    **options: dict\n) -&gt; ValuationResult\n</code></pre> <p>Implements group testing for approximation of Shapley values as described in (Jia, R. et al., 2019)<sup>1</sup>.</p> <p>Warning</p> <p>This method is very inefficient. It requires several orders of magnitude more evaluations of the utility than others in montecarlo. It also uses several intermediate objects like the results from the runners and the constraint matrices which can become rather large.</p> <p>By picking a specific distribution over subsets, the differences in Shapley values can be approximated with a Monte Carlo sum. These are then used to solve for the individual values in a feasibility problem.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Number of tests to perform. Use num_samples_eps_delta to estimate this.</p> <p> TYPE: <code>int</code> </p> <code>epsilon</code> <p>From the (\u03b5,\u03b4) sample bound. Use the same as for the estimation of <code>n_iterations</code>.</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>From the (\u03b5,\u03b4) sample bound. Use the same as for the estimation of <code>n_iterations</code>.</p> <p> TYPE: <code>float</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use. Each worker performs a chunk of all tests (i.e. utility evaluations).</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>options</code> <p>Additional options to pass to cvxpy.Problem.solve(). E.g. to change the solver (which defaults to <code>cvxpy.SCS</code>) pass <code>solver=cvxpy.CVXOPT</code>.</p> <p> TYPE: <code>dict</code> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.5.0</p> <p>Changed the solver to cvxpy instead of scipy's linprog. Added the ability to pass arbitrary options to it.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/shapley/gt.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef group_testing_shapley(\n    u: Utility,\n    n_samples: int,\n    epsilon: float,\n    delta: float,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n    **options: dict,\n) -&gt; ValuationResult:\n    \"\"\"Implements group testing for approximation of Shapley values as described\n    in (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n\n    !!! Warning\n        This method is very inefficient. It requires several orders of magnitude\n        more evaluations of the utility than others in\n        [montecarlo][pydvl.value.shapley.montecarlo]. It also uses several intermediate\n        objects like the results from the runners and the constraint matrices\n        which can become rather large.\n\n    By picking a specific distribution over subsets, the differences in Shapley\n    values can be approximated with a Monte Carlo sum. These are then used to\n    solve for the individual values in a feasibility problem.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_samples: Number of tests to perform. Use\n            [num_samples_eps_delta][pydvl.value.shapley.gt.num_samples_eps_delta]\n            to estimate this.\n        epsilon: From the (\u03b5,\u03b4) sample bound. Use the same as for the\n            estimation of `n_iterations`.\n        delta: From the (\u03b5,\u03b4) sample bound. Use the same as for the\n            estimation of `n_iterations`.\n        n_jobs: Number of parallel jobs to use. Each worker performs a chunk\n            of all tests (i.e. utility evaluations).\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        options: Additional options to pass to\n            [cvxpy.Problem.solve()](https://www.cvxpy.org/tutorial/advanced/index.html#solve-method-options).\n            E.g. to change the solver (which defaults to `cvxpy.SCS`) pass\n            `solver=cvxpy.CVXOPT`.\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"New in version 0.4.0\"\n\n    !!! tip \"Changed in version 0.5.0\"\n        Changed the solver to cvxpy instead of scipy's linprog. Added the ability\n        to pass arbitrary options to it.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n\n    n = len(u.data.indices)\n\n    const = _constants(\n        n=n,\n        epsilon=epsilon,\n        delta=delta,\n        utility_range=u.score_range.max() - u.score_range.min(),\n    )\n    T = n_samples\n    if T &lt; const.T:\n        log.warning(\n            f\"n_samples of {T} are below the required {const.T} for the \"\n            f\"\u03b5={epsilon:.02f} guarantee at \u03b4={1 - delta:.02f} probability\"\n        )\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    samples_per_job = max(1, n_samples // parallel_backend.effective_n_jobs(n_jobs))\n\n    def reducer(\n        results_it: Iterable[Tuple[NDArray, NDArray]]\n    ) -&gt; Tuple[NDArray, NDArray]:\n        return np.concatenate(list(x[0] for x in results_it)).astype(\n            np.float_\n        ), np.concatenate(list(x[1] for x in results_it)).astype(np.int_)\n\n    seed_sequence = ensure_seed_sequence(seed)\n    map_reduce_seed_sequence, cvxpy_seed = tuple(seed_sequence.spawn(2))\n\n    map_reduce_job: MapReduceJob[Utility, Tuple[NDArray, NDArray]] = MapReduceJob(\n        u,\n        map_func=_group_testing_shapley,\n        reduce_func=reducer,\n        map_kwargs=dict(n_samples=samples_per_job, progress=progress),\n        parallel_backend=parallel_backend,\n        n_jobs=n_jobs,\n    )\n    uu, betas = map_reduce_job(seed=map_reduce_seed_sequence)\n\n    # Matrix of estimated differences. See Eqs. (3) and (4) in the paper.\n    C = np.zeros(shape=(n, n))\n    for i in range(n):\n        for j in range(i + 1, n):\n            C[i, j] = np.dot(uu, betas[:, i] - betas[:, j])\n    C *= const.Z / T\n    total_utility = u(u.data.indices)\n\n    ###########################################################################\n    # Solution of the constraint problem with cvxpy\n\n    v = cp.Variable(n)\n    constraints = [cp.sum(v) == total_utility]\n    for i in range(n):\n        for j in range(i + 1, n):\n            constraints.append(v[i] - v[j] &lt;= epsilon + C[i, j])\n            constraints.append(v[j] - v[i] &lt;= epsilon - C[i, j])\n\n    problem = cp.Problem(cp.Minimize(0), constraints)\n    solver = options.pop(\"solver\", cp.SCS)\n    problem.solve(solver=solver, **options)\n\n    if problem.status != \"optimal\":\n        log.warning(f\"cvxpy returned status {problem.status}\")\n        values = (\n            np.nan * np.ones_like(u.data.indices)\n            if not hasattr(v.value, \"__len__\")\n            else v.value\n        )\n        status = Status.Failed\n    else:\n        values = v.value\n        status = Status.Converged\n\n    return ValuationResult(\n        algorithm=\"group_testing_shapley\",\n        status=status,\n        values=values,\n        data_names=u.data.data_names,\n        solver_status=problem.status,\n    )\n</code></pre>"},{"location":"api/pydvl/value/shapley/knn/","title":"Knn","text":""},{"location":"api/pydvl/value/shapley/knn/#pydvl.value.shapley.knn","title":"pydvl.value.shapley.knn","text":"<p>This module contains Shapley computations for K-Nearest Neighbours.</p> <p>Todo</p> <p>Implement approximate KNN computation for sublinear complexity</p>"},{"location":"api/pydvl/value/shapley/knn/#pydvl.value.shapley.knn--references","title":"References","text":"<ol> <li> <p>Jia, R. et al., 2019. Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms. In: Proceedings of the VLDB Endowment, Vol. 12, No. 11, pp. 1610\u20131623.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/knn/#pydvl.value.shapley.knn.knn_shapley","title":"knn_shapley","text":"<pre><code>knn_shapley(u: Utility, *, progress: bool = True) -&gt; ValuationResult\n</code></pre> <p>Computes exact Shapley values for a KNN classifier.</p> <p>This implements the method described in (Jia, R. et al., 2019)<sup>1</sup>. It exploits the local structure of K-Nearest Neighbours to reduce the number of calls to the utility function to a constant number per index, thus reducing computation time to \\(O(n)\\).</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility with a KNN model to extract parameters from. The object will not be modified nor used other than to call get_params()</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> RAISES DESCRIPTION <code>TypeError</code> <p>If the model in the utility is not a sklearn.neighbors.KNeighborsClassifier.</p> <p>New in version 0.1.0</p> Source code in <code>src/pydvl/value/shapley/knn.py</code> <pre><code>def knn_shapley(u: Utility, *, progress: bool = True) -&gt; ValuationResult:\n    \"\"\"Computes exact Shapley values for a KNN classifier.\n\n    This implements the method described in (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019a\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    It exploits the local structure of K-Nearest Neighbours to reduce the number\n    of calls to the utility function to a constant number per index, thus\n    reducing computation time to $O(n)$.\n\n    Args:\n        u: Utility with a KNN model to extract parameters from. The object\n            will not be modified nor used other than to call [get_params()](\n            &lt;https://scikit-learn.org/stable/modules/generated/sklearn.base.BaseEstimator.html#sklearn.base.BaseEstimator.get_params&gt;)\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the data values.\n\n    Raises:\n        TypeError: If the model in the utility is not a\n            [sklearn.neighbors.KNeighborsClassifier][].\n\n    !!! tip \"New in version 0.1.0\"\n\n    \"\"\"\n    if not isinstance(u.model, KNeighborsClassifier):\n        raise TypeError(\"KNN Shapley requires a K-Nearest Neighbours model\")\n\n    defaults: Dict[str, Union[int, str]] = {\n        \"algorithm\": \"ball_tree\" if u.data.dim &gt;= 20 else \"kd_tree\",\n        \"metric\": \"minkowski\",\n        \"p\": 2,\n    }\n    defaults.update(u.model.get_params())\n    # HACK: NearestNeighbors doesn't support this. There will be more...\n    del defaults[\"weights\"]\n    n_neighbors: int = int(defaults[\"n_neighbors\"])\n    defaults[\"n_neighbors\"] = len(u.data)  # We want all training points sorted\n\n    assert n_neighbors &lt; len(u.data)\n    # assert data.target_dim == 1\n\n    nns = NearestNeighbors(**defaults).fit(u.data.x_train)\n    # closest to farthest\n    _, indices = nns.kneighbors(u.data.x_test)\n\n    values: NDArray[np.float_] = np.zeros_like(u.data.indices, dtype=np.float_)\n    n = len(u.data)\n    yt = u.data.y_train\n    iterator = enumerate(zip(u.data.y_test, indices), start=1)\n    for j, (y, ii) in tqdm(iterator, disable=not progress):\n        value_at_x = int(yt[ii[-1]] == y) / n\n        values[ii[-1]] += (value_at_x - values[ii[-1]]) / j\n        for i in range(n - 2, n_neighbors, -1):  # farthest to closest\n            value_at_x = (\n                values[ii[i + 1]] + (int(yt[ii[i]] == y) - int(yt[ii[i + 1]] == y)) / i\n            )\n            values[ii[i]] += (value_at_x - values[ii[i]]) / j\n        for i in range(n_neighbors, -1, -1):  # farthest to closest\n            value_at_x = (\n                values[ii[i + 1]]\n                + (int(yt[ii[i]] == y) - int(yt[ii[i + 1]] == y)) / n_neighbors\n            )\n            values[ii[i]] += (value_at_x - values[ii[i]]) / j\n\n    return ValuationResult(\n        algorithm=\"knn_shapley\",\n        status=Status.Converged,\n        values=values,\n        data_names=u.data.data_names,\n    )\n</code></pre>"},{"location":"api/pydvl/value/shapley/montecarlo/","title":"Montecarlo","text":""},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo","title":"pydvl.value.shapley.montecarlo","text":"<p>Monte Carlo approximations to Shapley Data values.</p> <p>Warning</p> <p>You probably want to use the common interface provided by compute_shapley_values() instead of directly using the functions in this module.</p> <p>Because exact computation of Shapley values requires \\(\\mathcal{O}(2^n)\\) re-trainings of the model, several Monte Carlo approximations are available. The first two sample from the powerset of the training data directly: combinatorial_montecarlo_shapley() and owen_sampling_shapley(). The latter uses a reformulation in terms of a continuous extension of the utility.</p> <p>Alternatively, employing another reformulation of the expression above as a sum over permutations, one has the implementation in permutation_montecarlo_shapley() with the option to pass an early stopping strategy to reduce computation as done in Truncated MonteCarlo Shapley (TMCS).</p> <p>Also see</p> <p>It is also possible to use group_testing_shapley() to reduce the number of evaluations of the utility. The method is however typically outperformed by others in this module.</p> <p>Also see</p> <p>Additionally, you can consider grouping your data points using GroupedDataset and computing the values of the groups instead. This is not to be confused with \"group testing\" as implemented in group_testing_shapley(): any of the algorithms mentioned above, including Group Testing, can work to valuate groups of samples as units.</p>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley","title":"permutation_montecarlo_shapley","text":"<pre><code>permutation_montecarlo_shapley(\n    u: Utility,\n    done: StoppingCriterion,\n    *,\n    truncation: TruncationPolicy = NoTruncation(),\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes an approximate Shapley value by sampling independent permutations of the index set, approximating the sum:</p> \\[ v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} \\tilde{w}( | \\sigma_{:i} | )[u(\\sigma_{:i} \\cup \\{i\\}) \u2212 u(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears (see [[data-valuation]] for details).</p> <p>This implements the method described in (Ghorbani and Zou, 2019)<sup>1</sup> with a double stopping criterion.</p> <p>Todo</p> <p>Think of how to add Robin-Gelman or some other more principled stopping criterion.</p> <p>Instead of naively implementing the expectation, we sequentially add points to coalitions from a permutation and incrementally compute marginal utilities. We stop computing marginals for a given permutation based on a TruncationPolicy. (Ghorbani and Zou, 2019)<sup>1</sup> mention two policies: one that stops after a certain fraction of marginals are computed, implemented in FixedTruncation, and one that stops if the last computed utility (\"score\") is close to the total utility using the standard deviation of the utility as a measure of proximity, implemented in BootstrapTruncation.</p> <p>We keep sampling permutations and updating all shapley values until the StoppingCriterion returns <code>True</code>.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>function checking whether computation must stop.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>truncation</code> <p>An optional callable which decides whether to interrupt processing a permutation and set all subsequent marginals to zero. Typically used to stop computation when the marginal is small.</p> <p> TYPE: <code>TruncationPolicy</code> DEFAULT: <code>NoTruncation()</code> </p> <code>n_jobs</code> <p>number of jobs across which to distribute the computation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/shapley/montecarlo.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef permutation_montecarlo_shapley(\n    u: Utility,\n    done: StoppingCriterion,\n    *,\n    truncation: TruncationPolicy = NoTruncation(),\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    r\"\"\"Computes an approximate Shapley value by sampling independent\n    permutations of the index set, approximating the sum:\n\n    $$\n    v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)}\n    \\tilde{w}( | \\sigma_{:i} | )[u(\\sigma_{:i} \\cup \\{i\\}) \u2212 u(\\sigma_{:i})],\n    $$\n\n    where $\\sigma_{:i}$ denotes the set of indices in permutation sigma before\n    the position where $i$ appears (see [[data-valuation]] for details).\n\n    This implements the method described in (Ghorbani and Zou, 2019)&lt;sup&gt;&lt;a\n    href=\"#ghorbani_data_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt; with a double stopping criterion.\n\n    !!! Todo\n        Think of how to add Robin-Gelman or some other more principled stopping\n        criterion.\n\n    Instead of naively implementing the expectation, we sequentially add points\n    to coalitions from a permutation and incrementally compute marginal utilities.\n    We stop computing marginals for a given permutation based on a\n    [TruncationPolicy][pydvl.value.shapley.truncated.TruncationPolicy].\n    (Ghorbani and Zou, 2019)&lt;sup&gt;&lt;a href=\"#ghorbani_data_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n    mention two policies: one that stops after a certain\n    fraction of marginals are computed, implemented in\n    [FixedTruncation][pydvl.value.shapley.truncated.FixedTruncation],\n    and one that stops if the last computed utility (\"score\") is close to the\n    total utility using the standard deviation of the utility as a measure of\n    proximity, implemented in\n    [BootstrapTruncation][pydvl.value.shapley.truncated.BootstrapTruncation].\n\n    We keep sampling permutations and updating all shapley values\n    until the [StoppingCriterion][pydvl.value.stopping.StoppingCriterion] returns\n    `True`.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: function checking whether computation must stop.\n        truncation: An optional callable which decides whether to interrupt\n            processing a permutation and set all subsequent marginals to\n            zero. Typically used to stop computation when the marginal is small.\n        n_jobs: number of jobs across which to distribute the computation.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    algorithm = \"permutation_montecarlo_shapley\"\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n    u = parallel_backend.put(u)\n    max_workers = parallel_backend.effective_n_jobs(n_jobs)\n    n_submitted_jobs = 2 * max_workers  # number of jobs in the executor's queue\n\n    seed_sequence = ensure_seed_sequence(seed)\n    result = ValuationResult.zeros(\n        algorithm=algorithm, indices=u.data.indices, data_names=u.data.data_names\n    )\n\n    pbar = tqdm(disable=not progress, total=100, unit=\"%\")\n\n    with parallel_backend.executor(\n        max_workers=max_workers, cancel_futures=CancellationPolicy.ALL\n    ) as executor:\n        pending: set[Future] = set()\n        while True:\n            pbar.n = 100 * done.completion()\n            pbar.refresh()\n\n            completed, pending = wait(pending, timeout=1.0, return_when=FIRST_COMPLETED)\n            for future in completed:\n                result += future.result()\n                # we could check outside the loop, but that means more\n                # submissions if the stopping criterion is unstable\n                if done(result):\n                    return result\n\n            # Ensure that we always have n_submitted_jobs in the queue or running\n            n_remaining_slots = n_submitted_jobs - len(pending)\n            seeds = seed_sequence.spawn(n_remaining_slots)\n            for i in range(n_remaining_slots):\n                future = executor.submit(\n                    _permutation_montecarlo_one_step,\n                    u,\n                    truncation,\n                    algorithm,\n                    seed=seeds[i],\n                )\n                pending.add(future)\n</code></pre>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley","title":"combinatorial_montecarlo_shapley","text":"<pre><code>combinatorial_montecarlo_shapley(\n    u: Utility,\n    done: StoppingCriterion,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes an approximate Shapley value using the combinatorial definition:</p> \\[v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)]\\] <p>This consists of randomly sampling subsets of the power set of the training indices in u.data, and computing their marginal utilities. See Data valuation for details.</p> <p>Note that because sampling is done with replacement, the approximation is poor even for \\(2^{m}\\) subsets with \\(m&gt;n\\), even though there are \\(2^{n-1}\\) subsets for each \\(i\\). Prefer permutation_montecarlo_shapley().</p> <p>Parallelization is done by splitting the set of indices across processes and computing the sum over subsets \\(S \\subseteq N \\setminus \\{i\\}\\) separately.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion for the computation.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>n_jobs</code> <p>number of parallel jobs across which to distribute the computation. Each worker receives a chunk of indices</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/shapley/montecarlo.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef combinatorial_montecarlo_shapley(\n    u: Utility,\n    done: StoppingCriterion,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    r\"\"\"Computes an approximate Shapley value using the combinatorial\n    definition:\n\n    $$v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}}\n    \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)]$$\n\n    This consists of randomly sampling subsets of the power set of the training\n    indices in [u.data][pydvl.utils.utility.Utility], and computing their\n    marginal utilities. See [Data valuation][data-valuation] for details.\n\n    Note that because sampling is done with replacement, the approximation is\n    poor even for $2^{m}$ subsets with $m&gt;n$, even though there are $2^{n-1}$\n    subsets for each $i$. Prefer\n    [permutation_montecarlo_shapley()][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].\n\n    Parallelization is done by splitting the set of indices across processes and\n    computing the sum over subsets $S \\subseteq N \\setminus \\{i\\}$ separately.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        done: Stopping criterion for the computation.\n        n_jobs: number of parallel jobs across which to distribute the\n            computation. Each worker receives a chunk of\n            [indices][pydvl.utils.dataset.Dataset.indices]\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    map_reduce_job: MapReduceJob[NDArray, ValuationResult] = MapReduceJob(\n        u.data.indices,\n        map_func=_combinatorial_montecarlo_shapley,\n        reduce_func=lambda results: reduce(operator.add, results),\n        map_kwargs=dict(u=u, done=done, progress=progress),\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n    )\n    return map_reduce_job(seed=seed)\n</code></pre>"},{"location":"api/pydvl/value/shapley/naive/","title":"Naive","text":""},{"location":"api/pydvl/value/shapley/naive/#pydvl.value.shapley.naive","title":"pydvl.value.shapley.naive","text":"<p>This module implements exact Shapley values using either the combinatorial or permutation definition.</p> <p>The exact computation of \\(n\\) values takes \\(\\mathcal{O}(2^n)\\) evaluations of the utility and is therefore only possible for small datasets. For larger datasets, consider using any of the approximations, such as Monte Carlo, or proxy models like kNN.</p> <p>See Data valuation for details.</p>"},{"location":"api/pydvl/value/shapley/naive/#pydvl.value.shapley.naive.permutation_exact_shapley","title":"permutation_exact_shapley","text":"<pre><code>permutation_exact_shapley(\n    u: Utility, *, progress: bool = True\n) -&gt; ValuationResult\n</code></pre> <p>Computes the exact Shapley value using the formulation with permutations:</p> \\[v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{i-1} \\cup {i}) \u2212 u(\\sigma_{i})].\\] <p>See Data valuation for details.</p> <p>When the length of the training set is &gt; 10 this prints a warning since the computation becomes too expensive. Used mostly for internal testing and simple use cases. Please refer to the Monte Carlo approximations for practical applications.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/naive.py</code> <pre><code>def permutation_exact_shapley(u: Utility, *, progress: bool = True) -&gt; ValuationResult:\n    r\"\"\"Computes the exact Shapley value using the formulation with permutations:\n\n    $$v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{i-1}\n    \\cup {i}) \u2212 u(\\sigma_{i})].$$\n\n    See [Data valuation][data-valuation] for details.\n\n    When the length of the training set is &gt; 10 this prints a warning since the\n    computation becomes too expensive. Used mostly for internal testing and\n    simple use cases. Please refer to the [Monte Carlo\n    approximations][pydvl.value.shapley.montecarlo] for practical applications.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        progress: Whether to display progress bars for each job.\n\n    Returns:\n        Object with the data values.\n    \"\"\"\n\n    n = len(u.data)\n    # Note that the cache in utility saves most of the refitting because we\n    # use frozenset for the input.\n    if n &gt; 10:\n        warnings.warn(\n            f\"Large dataset! Computation requires {n}! calls to utility()\",\n            RuntimeWarning,\n        )\n\n    values = np.zeros(n)\n    for p in tqdm(\n        permutations(u.data.indices),\n        disable=not progress,\n        desc=\"Permutation\",\n        total=math.factorial(n),\n    ):\n        for i, idx in enumerate(p):\n            values[idx] += u(p[: i + 1]) - u(p[:i])\n    values /= math.factorial(n)\n\n    return ValuationResult(\n        algorithm=\"permutation_exact_shapley\",\n        status=Status.Converged,\n        values=values,\n        data_names=u.data.data_names,\n    )\n</code></pre>"},{"location":"api/pydvl/value/shapley/naive/#pydvl.value.shapley.naive.combinatorial_exact_shapley","title":"combinatorial_exact_shapley","text":"<pre><code>combinatorial_exact_shapley(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False\n) -&gt; ValuationResult\n</code></pre> <p>Computes the exact Shapley value using the combinatorial definition.</p> \\[v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)].\\] <p>See Data valuation for details.</p> <p>Note</p> <p>If the length of the training set is &gt; n_jobs*20 this prints a warning because the computation is very expensive. Used mostly for internal testing and simple use cases. Please refer to the Monte Carlo approximations for practical applications.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/shapley/naive.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef combinatorial_exact_shapley(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n) -&gt; ValuationResult:\n    r\"\"\"Computes the exact Shapley value using the combinatorial definition.\n\n    $$v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}}\n    \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)].$$\n\n    See [Data valuation][data-valuation] for details.\n\n    !!! Note\n        If the length of the training set is &gt; n_jobs*20 this prints a warning\n        because the computation is very expensive. Used mostly for internal\n        testing and simple use cases. Please refer to the\n        [Monte Carlo][pydvl.value.shapley.montecarlo] approximations for\n        practical applications.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_jobs: Number of parallel jobs to use\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    # Arbitrary choice, will depend on time required, caching, etc.\n    if len(u.data) // n_jobs &gt; 20:\n        warnings.warn(\n            f\"Large dataset! Computation requires 2^{len(u.data)} calls to model.fit()\"\n        )\n\n    def reduce_fun(results: List[NDArray]) -&gt; NDArray:\n        return np.array(results).sum(axis=0)  # type: ignore\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    map_reduce_job: MapReduceJob[NDArray, NDArray] = MapReduceJob(\n        u.data.indices,\n        map_func=_combinatorial_exact_shapley,\n        map_kwargs=dict(u=u, progress=progress),\n        reduce_func=reduce_fun,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n    )\n    values = map_reduce_job()\n    return ValuationResult(\n        algorithm=\"combinatorial_exact_shapley\",\n        status=Status.Converged,\n        values=values,\n        data_names=u.data.data_names,\n    )\n</code></pre>"},{"location":"api/pydvl/value/shapley/owen/","title":"Owen","text":""},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen","title":"pydvl.value.shapley.owen","text":""},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen--references","title":"References","text":"<ol> <li> <p>Okhrati, R., Lipani, A., 2021. A Multilinear Sampling Algorithm to Estimate Shapley Values. In: 2020 25th International Conference on Pattern Recognition (ICPR), pp. 7992\u20137999. IEEE.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen.OwenAlgorithm","title":"OwenAlgorithm","text":"<p>             Bases: <code>Enum</code></p> <p>Choices for the Owen sampling method.</p> ATTRIBUTE DESCRIPTION <code>Standard</code> <p>Use q \u2208 [0, 1]</p> <p> </p> <code>Antithetic</code> <p>Use q \u2208 [0, 0.5] and correlated samples</p> <p> </p>"},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen.owen_sampling_shapley","title":"owen_sampling_shapley","text":"<pre><code>owen_sampling_shapley(\n    u: Utility,\n    n_samples: int,\n    max_q: int,\n    *,\n    method: OwenAlgorithm = OwenAlgorithm.Standard,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Owen sampling of Shapley values as described in (Okhrati and Lipani, 2021)<sup>1</sup>.</p> <p>This function computes a Monte Carlo approximation to</p> \\[v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{\\backslash \\{i\\}})} [u(S \\cup \\{i\\}) - u(S)]\\] <p>using one of two methods. The first one, selected with the argument <code>mode = OwenAlgorithm.Standard</code>, approximates the integral with:</p> \\[\\hat{v}_u(i) = \\frac{1}{Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m \\cup \\{i\\}) - u(S^{(q_j)}_m)],\\] <p>where \\(q_j = \\frac{j}{Q} \\in [0,1]\\) and the sets \\(S^{(q_j)}\\) are such that a sample \\(x \\in S^{(q_j)}\\) if a draw from a \\(Ber(q_j)\\) distribution is 1.</p> <p>The second method, selected with the argument <code>mode = OwenAlgorithm.Antithetic</code>, uses correlated samples in the inner sum to reduce the variance:</p> \\[\\hat{v}_u(i) = \\frac{1}{2 Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m \\cup \\{i\\}) - u(S^{(q_j)}_m) + u((S^{(q_j)}_m)^c \\cup \\{i\\}) - u((S^{( q_j)}_m)^c)],\\] <p>where now \\(q_j = \\frac{j}{2Q} \\in [0,\\frac{1}{2}]\\), and \\(S^c\\) is the complement of \\(S\\).</p> <p>Note</p> <p>The outer integration could be done instead with a quadrature rule.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object holding data, model and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Numer of sets to sample for each value of q</p> <p> TYPE: <code>int</code> </p> <code>max_q</code> <p>Number of subdivisions for q \u2208 [0,1] (the element sampling probability) used to approximate the outer integral.</p> <p> TYPE: <code>int</code> </p> <code>method</code> <p>Selects the algorithm to use, see the description. Either OwenAlgorithm.Full for \\(q \\in [0,1]\\) or OwenAlgorithm.Halved for \\(q \\in [0,0.5]\\) and correlated samples</p> <p> TYPE: <code>OwenAlgorithm</code> DEFAULT: <code>Standard</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use. Each worker receives a chunk of the total of <code>max_q</code> values for q.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.3.0</p> <p>Changed in version 0.5.0</p> <p>Support for parallel computation and enable antithetic sampling.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/shapley/owen.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef owen_sampling_shapley(\n    u: Utility,\n    n_samples: int,\n    max_q: int,\n    *,\n    method: OwenAlgorithm = OwenAlgorithm.Standard,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult:\n    r\"\"\"Owen sampling of Shapley values as described in\n    (Okhrati and Lipani, 2021)&lt;sup&gt;&lt;a href=\"#okhrati_multilinear_2021\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n\n    This function computes a Monte Carlo approximation to\n\n    $$v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{\\backslash \\{i\\}})}\n    [u(S \\cup \\{i\\}) - u(S)]$$\n\n    using one of two methods. The first one, selected with the argument ``mode =\n    OwenAlgorithm.Standard``, approximates the integral with:\n\n    $$\\hat{v}_u(i) = \\frac{1}{Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m\n    \\cup \\{i\\}) - u(S^{(q_j)}_m)],$$\n\n    where $q_j = \\frac{j}{Q} \\in [0,1]$ and the sets $S^{(q_j)}$ are such that a\n    sample $x \\in S^{(q_j)}$ if a draw from a $Ber(q_j)$ distribution is 1.\n\n    The second method, selected with the argument ``mode =\n    OwenAlgorithm.Antithetic``, uses correlated samples in the inner sum to\n    reduce the variance:\n\n    $$\\hat{v}_u(i) = \\frac{1}{2 Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m\n    \\cup \\{i\\}) - u(S^{(q_j)}_m) + u((S^{(q_j)}_m)^c \\cup \\{i\\}) - u((S^{(\n    q_j)}_m)^c)],$$\n\n    where now $q_j = \\frac{j}{2Q} \\in [0,\\frac{1}{2}]$, and $S^c$ is the\n    complement of $S$.\n\n    !!! Note\n        The outer integration could be done instead with a quadrature rule.\n\n    Args:\n        u: [Utility][pydvl.utils.utility.Utility] object holding data, model\n            and scoring function.\n        n_samples: Numer of sets to sample for each value of q\n        max_q: Number of subdivisions for q \u2208 [0,1] (the element sampling\n            probability) used to approximate the outer integral.\n        method: Selects the algorithm to use, see the description. Either\n            [OwenAlgorithm.Full][pydvl.value.shapley.owen.OwenAlgorithm] for\n            $q \\in [0,1]$ or\n            [OwenAlgorithm.Halved][pydvl.value.shapley.owen.OwenAlgorithm] for\n            $q \\in [0,0.5]$ and correlated samples\n        n_jobs: Number of parallel jobs to use. Each worker receives a chunk\n            of the total of `max_q` values for q.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"New in version 0.3.0\"\n\n    !!! tip \"Changed in version 0.5.0\"\n        Support for parallel computation and enable antithetic sampling.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n\n    \"\"\"\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    map_reduce_job: MapReduceJob[NDArray, ValuationResult] = MapReduceJob(\n        u.data.indices,\n        map_func=_owen_sampling_shapley,\n        reduce_func=lambda results: reduce(operator.add, results),\n        map_kwargs=dict(\n            u=u,\n            method=OwenAlgorithm(method),\n            n_samples=n_samples,\n            max_q=max_q,\n            progress=progress,\n        ),\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n    )\n\n    return map_reduce_job(seed=seed)\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/","title":"Truncated","text":""},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated","title":"pydvl.value.shapley.truncated","text":""},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy","title":"TruncationPolicy","text":"<pre><code>TruncationPolicy()\n</code></pre> <p>             Bases: <code>ABC</code></p> <p>A policy for deciding whether to stop computing marginals in a permutation.</p> <p>Statistics are kept on the number of calls and truncations as <code>n_calls</code> and <code>n_truncations</code> respectively.</p> ATTRIBUTE DESCRIPTION <code>n_calls</code> <p>Number of calls to the policy.</p> <p> TYPE: <code>int</code> </p> <code>n_truncations</code> <p>Number of truncations made by the policy.</p> <p> TYPE: <code>int</code> </p> <p>Todo</p> <p>Because the policy objects are copied to the workers, the statistics are not accessible from the coordinating process. We need to add methods for this.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self) -&gt; None:\n    self.n_calls: int = 0\n    self.n_truncations: int = 0\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy.reset","title":"reset  <code>abstractmethod</code>","text":"<pre><code>reset(u: Optional[Utility] = None)\n</code></pre> <p>Reset the policy to a state ready for a new permutation.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>@abc.abstractmethod\ndef reset(self, u: Optional[Utility] = None):\n    \"\"\"Reset the policy to a state ready for a new permutation.\"\"\"\n    ...\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy.__call__","title":"__call__","text":"<pre><code>__call__(idx: int, score: float) -&gt; bool\n</code></pre> <p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n    \"\"\"Check whether the computation should be interrupted.\n\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\n    ret = self._check(idx, score)\n    self.n_calls += 1\n    self.n_truncations += 1 if ret else 0\n    return ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.NoTruncation","title":"NoTruncation","text":"<pre><code>NoTruncation()\n</code></pre> <p>             Bases: <code>TruncationPolicy</code></p> <p>A policy which never interrupts the computation.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self) -&gt; None:\n    self.n_calls: int = 0\n    self.n_truncations: int = 0\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.NoTruncation.__call__","title":"__call__","text":"<pre><code>__call__(idx: int, score: float) -&gt; bool\n</code></pre> <p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n    \"\"\"Check whether the computation should be interrupted.\n\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\n    ret = self._check(idx, score)\n    self.n_calls += 1\n    self.n_truncations += 1 if ret else 0\n    return ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.FixedTruncation","title":"FixedTruncation","text":"<pre><code>FixedTruncation(u: Utility, fraction: float)\n</code></pre> <p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation after computing a fixed number of marginals.</p> <p>The experiments in Appendix B of (Ghorbani and Zou, 2019)<sup>1</sup> show that when the training set size is large enough, one can simply truncate the iteration over permutations after a fixed number of steps. This happens because beyond a certain number of samples in a training set, the model becomes insensitive to new ones. Alas, this strongly depends on the data distribution and the model and there is no automatic way of estimating this number.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>fraction</code> <p>Fraction of marginals in a permutation to compute before stopping (e.g. 0.5 to compute half of the marginals).</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, fraction: float):\n    super().__init__()\n    if fraction &lt;= 0 or fraction &gt; 1:\n        raise ValueError(\"fraction must be in (0, 1]\")\n    self.max_marginals = len(u.data) * fraction\n    self.count = 0\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.FixedTruncation.__call__","title":"__call__","text":"<pre><code>__call__(idx: int, score: float) -&gt; bool\n</code></pre> <p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n    \"\"\"Check whether the computation should be interrupted.\n\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\n    ret = self._check(idx, score)\n    self.n_calls += 1\n    self.n_truncations += 1 if ret else 0\n    return ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.RelativeTruncation","title":"RelativeTruncation","text":"<pre><code>RelativeTruncation(u: Utility, rtol: float)\n</code></pre> <p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation if the marginal utility is too low.</p> <p>This is called \"performance tolerance\" in (Ghorbani and Zou, 2019)<sup>1</sup>.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>rtol</code> <p>Relative tolerance. The permutation is broken if the last computed utility is less than <code>total_utility * rtol</code>.</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, rtol: float):\n    super().__init__()\n    self.rtol = rtol\n    logger.info(\"Computing total utility for permutation truncation.\")\n    self.total_utility = self.reset(u)\n    self._u = u\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.RelativeTruncation.__call__","title":"__call__","text":"<pre><code>__call__(idx: int, score: float) -&gt; bool\n</code></pre> <p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n    \"\"\"Check whether the computation should be interrupted.\n\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\n    ret = self._check(idx, score)\n    self.n_calls += 1\n    self.n_truncations += 1 if ret else 0\n    return ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.BootstrapTruncation","title":"BootstrapTruncation","text":"<pre><code>BootstrapTruncation(u: Utility, n_samples: int, sigmas: float = 1)\n</code></pre> <p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation if the last computed utility is close to the total utility, measured as a multiple of the standard deviation of the utilities.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Number of bootstrap samples to use to compute the variance of the utilities.</p> <p> TYPE: <code>int</code> </p> <code>sigmas</code> <p>Number of standard deviations to use as a threshold.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, n_samples: int, sigmas: float = 1):\n    super().__init__()\n    self.n_samples = n_samples\n    logger.info(\"Computing total utility for permutation truncation.\")\n    self.total_utility = u(u.data.indices)\n    self.count: int = 0\n    self.variance: float = 0\n    self.mean: float = 0\n    self.sigmas: float = sigmas\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.BootstrapTruncation.__call__","title":"__call__","text":"<pre><code>__call__(idx: int, score: float) -&gt; bool\n</code></pre> <p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n    \"\"\"Check whether the computation should be interrupted.\n\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\n    ret = self._check(idx, score)\n    self.n_calls += 1\n    self.n_truncations += 1 if ret else 0\n    return ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/types/","title":"Types","text":""},{"location":"api/pydvl/value/shapley/types/#pydvl.value.shapley.types","title":"pydvl.value.shapley.types","text":""},{"location":"api/pydvl/value/shapley/types/#pydvl.value.shapley.types.ShapleyMode","title":"ShapleyMode","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Supported algorithms for the computation of Shapley values.</p> <p>Todo</p> <p>Make algorithms register themselves here.</p>"},{"location":"examples/","title":"Examples","text":""},{"location":"examples/#data-valuation","title":"Data valuation","text":"<ul> <li> <p>Shapley values</p> <p>An introduction using the spotify dataset, showcasing grouped datasets and applied to improving model performance and identifying bogus data.</p> <p></p> </li> <li> <p>KNN Shapley</p> <p>A showcase of a fast model-specific valuation method using the iris dataset. </p> <p></p> </li> <li> <p>Data utility learning</p> <p>Learning a utility function from a few evaluations and using it to estimate the value of the remaining data.</p> <p></p> </li> <li> <p>Least Core</p> <p>An alternative solution concept from game theory, illustrated on a classification problem.</p> <p></p> </li> <li> <p>Data OOB</p> <p>A different and fast strategy for data valuation, using the out-of-bag error of a bagging model.</p> <p></p> </li> <li> <p>Faster Banzhaf values</p> <p>Using Banzhaf values to estimate the value of data points in MNIST, and evaluating convergence speed of MSR. </p> <p></p> </li> </ul>"},{"location":"examples/#influence-functions","title":"Influence functions","text":"<ul> <li> <p>For CNNs</p> <p>Detecting corrupted labels with influence functions on the ImageNet dataset.</p> <p></p> </li> <li> <p>For language models</p> <p>Using the IMDB dataset for sentiment analysis and a fine-tuned BERT model.</p> <p></p> </li> <li> <p>For mislabeled data</p> <p>Detecting corrupted labels using a synthetic dataset.</p> <p></p> </li> <li> <p>For outlier detection</p> <p>Using the wine dataset</p> <p></p> </li> </ul>"},{"location":"examples/data_oob/","title":"Data OOB","text":"<p>     This notebook introduces the Data-           OOB          method, an implementation based on a publication from Kwon and Zou \"           Data-             OOB            : Out-of-bag Estimate as a Simple and Efficient Data Value          \" ICML 2023 , using pyDVL.    </p> <p>     The objective of this paper is mainly to overcome the computational bottleneck of shapley-based data valuation methods that require to fit a significant number of models to accurately estimate marginal contributions. The algorithms compute data values from out of bag estimates using a bagging model.    </p> <p>     The value can be interpreted as a partition of the           OOB          estimate, which is originally introduced to estimate the prediction error. This           OOB          estimate is given as:    </p>      \\[ \\sum_{i=1}^n\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i, \\hat{f}_b(x_i))}{\\sum_{b=1}^{B} \\mathbb{1} (w_{bi}=0)} \\]     <pre><code>%autoreload\nfrom pydvl.utils import Dataset, Scorer, Seed, Utility, ensure_seed_sequence\nfrom pydvl.value import ValuationResult, compute_data_oob\n</code></pre> <p>     We will work with the           adult classification dataset          from the UCI repository. The objective is to predict whether a person earns more than 50k a year based on a set of features such as age, education, occupation, etc.    </p> <p>     With a helper function we download the data and obtain the following pandas dataframe, where the categorical features have been removed:    </p> <pre>\n<code>Found cached file: adult_data.pkl.\n</code>\n</pre> <pre><code>data_adult.head()\n</code></pre>            age                      fnlwgt                      education-num                      capital-gain                      capital-loss                      hours-per-week                      income                      0                      39                      77516                      13                      2174                      0                      40                      &lt;=50K                      1                      50                      83311                      13                      0                      0                      13                      &lt;=50K                      2                      38                      215646                      9                      0                      0                      40                      &lt;=50K                      3                      53                      234721                      7                      0                      0                      40                      &lt;=50K                      4                      28                      338409                      13                      0                      0                      40                      &lt;=50K           <pre><code>data = Dataset.from_arrays(\n    X=data_adult.drop(columns=[\"income\"]).values,\n    y=data_adult.loc[:, \"income\"].cat.codes.values,\n    random_state=random_state,\n)\n\nmodel = KNeighborsClassifier(n_neighbors=5)\n\nutility = Utility(model, data, Scorer(\"accuracy\", default=0.0))\n</code></pre> <pre><code>n_estimators = [100, 500]\noob_values = [\n    compute_data_oob(utility, n_est=n_est, max_samples=0.95, seed=random_state)\n    for n_est in n_estimators\n]\n</code></pre> <p>     The two results are stored in an array of           ValuationResult          objects. Here's their distribution. The left-hand side depicts value as it increases with rank and a 99% t-confidence interval. The right-hand side shows the histogram of values.    </p> <p>     Observe how adding estimators reduces the variance of the values, but doesn't change their distribution much.    </p>"},{"location":"examples/data_oob/#bagging-for-data-valuation","title":"Bagging for data valuation","text":""},{"location":"examples/data_oob/#setup","title":"Setup","text":"<p>     We begin by importing the main libraries and setting some defaults.    </p>      If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/data_oob/#computing-the-oob-values","title":"Computing the           OOB          values","text":"<p>     The main idea of Data-           OOB          is to take an existing classifier or regression model and compute a per-sample out-of-bag performance estimate via bagging.    </p> <p>     For this example, we use a simple KNN classifier with           \\(k=5\\)          neighbours on the data and compute the data-oob values with two choices for the number of estimators in the bagging. For that we construct a           Utility          object using the           Scorer          class to specify the metric to use for the evaluation. Note how we pass a random seed to           Dataset.from_arrays          in order to ensure that we always get the same split when running this notebook multiple times. This will be particularly important when running the standard point removal experiments later.    </p> <p>     We then use the           compute_data_oob          function to compute the data-oob values.    </p>"},{"location":"examples/data_oob/#point-removal-experiments","title":"Point removal experiments","text":"<p>     The standard procedure for the evaluation of data valuation schemes is the point removal experiment. The objective is to measure the evolution of performance when the best/worst points are removed from the training set. This can be done with the function           compute_removal_score          , which takes precomputed values and computes the performance of the model as points are removed.    </p> <p>     In order to test the true performance of DataOOB, we repeat the whole task of computing the values and the point removal experiment multiple times, including the splitting of the dataset into training and valuation sets. It is important to remember to pass random state adequately for full reproducibility.    </p>"},{"location":"examples/influence_imagenet/","title":"For CNNs","text":"If you are reading this in the documentation, some boilerplate has been omitted for convenience.     <pre><code>from pydvl.influence.torch import CgInfluence\nfrom pydvl.reporting.plots import plot_influence_distribution_by_label\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score\n</code></pre> <pre><code>label_names = {90: \"tables\", 100: \"boats\"}\ntrain_ds, val_ds, test_ds = load_preprocess_imagenet(\n    train_size=0.8,\n    test_size=0.1,\n    keep_labels=label_names,\n    downsampling_ratio=1,\n)\n\nprint(\"Normalised image dtype:\", train_ds[\"normalized_images\"][0].dtype)\nprint(\"Label type:\", type(train_ds[\"labels\"][0]))\nprint(\"Image type:\", type(train_ds[\"images\"][0]))\ntrain_ds.info()\n</code></pre> <p>     Let's take a closer look at a few image samples    </p> <p>     Let's now further pre-process the data and prepare for model training. The helper function     <code>      process_io     </code>     converts the normalized images into tensors and the labels to the indices 0 and 1 to train the classifier.    </p> <pre><code>def process_io(df: pd.DataFrame, labels: dict) -&amp;gt; Tuple[torch.Tensor, torch.Tensor]:\n    x = df[\"normalized_images\"]\n    y = df[\"labels\"]\n    ds_label_to_model_label = {\n        ds_label: idx for idx, ds_label in enumerate(labels.values())\n    }\n    x_nn = torch.stack(x.tolist()).to(DEVICE)\n    y_nn = torch.tensor([ds_label_to_model_label[yi] for yi in y], device=DEVICE)\n    return x_nn, y_nn\n\n\ntrain_x, train_y = process_io(train_ds, label_names)\nval_x, val_y = process_io(val_ds, label_names)\ntest_x, test_y = process_io(test_ds, label_names)\n\nbatch_size = 768\ntrain_data = DataLoader(TensorDataset(train_x, train_y), batch_size=batch_size)\ntest_data = DataLoader(TensorDataset(test_x, test_y), batch_size=batch_size)\nval_data = DataLoader(TensorDataset(val_x, val_y), batch_size=batch_size)\n</code></pre> <pre><code>device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel_ft = new_resnet_model(output_size=len(label_names))\nmgr = TrainingManager(\n    \"model_ft\",\n    model_ft,\n    nn.CrossEntropyLoss(),\n    train_data,\n    val_data,\n    MODEL_PATH,\n    device=device,\n)\n# Set use_cache=False to retrain the model\ntrain_loss, val_loss = mgr.train(n_epochs=50, use_cache=True)\n</code></pre> <pre><code>plot_losses(Losses(train_loss, val_loss))\n</code></pre> <p>     The confusion matrix and           \\(F_1\\)          score look good, especially considering the low resolution of the images and their complexity (they contain different objects)    </p> <pre><code>pred_y_test = np.argmax(model_ft(test_x).cpu().detach(), axis=1).cpu()\nmodel_score = f1_score(test_y.cpu(), pred_y_test, average=\"weighted\")\n\ncm = confusion_matrix(test_y.cpu(), pred_y_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_names.values())\nprint(\"f1_score of model:\", model_score)\ndisp.plot();\n</code></pre> <pre>\n<code>f1_score of model: 0.9062805208898536\n</code>\n</pre> <pre><code>influence_model = CgInfluence(mgr.model, mgr.loss, hessian_reg, progress=True)\ninfluence_model = influence_model.fit(train_data)\n</code></pre> <p>     On the instantiated influence object, we can call the method           influences          , which takes some test data and some input dataset with labels (which typically is the training data, or a subset of it). The influence type will be     <code>      up     </code>     . The other option,     <code>      perturbation     </code>     , is beyond the scope of this notebook, but more info can be found in the notebook           using the Wine dataset          or in the documentation for pyDVL.    </p> <pre><code>influences = influence_model.influences(test_x, test_y, train_x, train_y, mode=\"up\")\n</code></pre> <p>     The output  is a matrix of size     <code>      test_set_length     </code>     x     <code>      training_set_length     </code>     . Each row represents a test data point, and each column a training data point, so that entry           \\((i,j)\\)          represents the influence of training point           \\(j\\)          on test point           \\(i\\)          .    </p> <p>     Now we plot the histogram of the influence that all training images have on the image selected above, separated by their label.    </p> <p>     Rather unsurprisingly, the training points with the highest influence have the same label. Now we can take the training images with the same label and show those with highest and lowest scores.    </p> <p>     Looking at the images, it is difficult to explain why those on the right are more influential than those on the left. At first sight, the choice seems to be random (or at the very least noisy). Let's dig in a bit more by looking at average influences:    </p> <pre><code>avg_influences = np.mean(influences.cpu().numpy(), axis=0)\n</code></pre> <p>     Once again, let's plot the histogram of influence values by label.    </p> <p>     Next, for each class (you can change value by changing label key) we can have a look at the top and bottom images by average influence, i.e. we can show the images that have the highest and lowest average influence over all test images.    </p> <p>     Once again, it is not easy to explain why the images on the left have a lower influence than the ones on the right.    </p> <pre><code>corrupted_model = new_resnet_model(output_size=len(label_names))\ncorrupted_dataset, corrupted_indices = corrupt_imagenet(\n    dataset=train_ds,\n    fraction_to_corrupt=0.1,\n    avg_influences=avg_influences,\n)\n\ncorrupted_train_x, corrupted_train_y = process_io(corrupted_dataset, label_names)\ncorrupted_data = DataLoader(\n    TensorDataset(corrupted_train_x, corrupted_train_y), batch_size=batch_size\n)\n\nmgr = TrainingManager(\n    \"corrupted_model\",\n    corrupted_model,\n    nn.CrossEntropyLoss(),\n    corrupted_data,\n    val_data,\n    MODEL_PATH,\n    device=device,\n)\ntraining_loss, validation_loss = mgr.train(n_epochs=50, use_cache=True)\n</code></pre> <pre><code>plot_losses(Losses(training_loss, validation_loss))\n</code></pre> <pre>\n<code>F1 score of model with corrupted data: 0.8541666666666666\n</code>\n</pre> <p>     Interestingly, despite being trained on a corrupted dataset, the model has a fairly high           \\(F_1\\)          score. Let's now calculate the influence of the corrupted training data points over the test data points.    </p> <pre><code>influence_model = CgInfluence(mgr.model, mgr.loss, hessian_reg, progress=True)\ninfluence_model = influence_model.fit(corrupted_data)\ninfluences = influence_model.influences(\n    test_x, test_y, corrupted_train_x, corrupted_train_y\n)\n</code></pre> <p>     As before, since we are interested in the average influence on the test dataset, we take the average of influences across rows, and then plot the highest and lowest influences for a chosen label    </p> <pre><code>avg_corrupted_influences = np.mean(influences.cpu().numpy(), axis=0)\n</code></pre> <p>     As expected, the samples with lowest (negative) influence for the label \"boats\" are those that have been corrupted: all the images on the left are tables! We can compare the average influence of corrupted data with non-corrupted ones    </p>            label                      avg_non_corrupted_infl                      avg_corrupted_infl                      score_diff                      0                      tables                      -0.405254                      -12.999691                      12.594438                      1                      boats                      -0.544211                      -13.080050                      12.535838           <p>     And indeed corrupted data have a more negative influence on average than clean ones!    </p> <p>     Despite this being a useful property, influence functions are known to be unreliable for tasks of data valuation, especially in deep learning where the fundamental assumption of the theory (convexity) is grossly violated. A lot of factors (e.g. the size of the network, the training process or the Hessian regularization term) can interfere with the computation, to the point that often the results that we obtain cannot be trusted. This has been extensively studied in the recent paper:    </p> <p>       Basu, S., P. Pope, and S. Feizi.             Influence Functions in Deep Learning Are Fragile.            International Conference on Learning Representations (ICLR). 2021          .    </p> <p>     Nevertheless, influence functions offer a relatively quick and mathematically rigorous way to evaluate (at first order) the importance of a training point for a model's prediction.    </p>"},{"location":"examples/influence_imagenet/#influence-functions-for-neural-networks","title":"Influence functions for neural networks","text":"<p>     This notebook explores the use of influence functions for convolutional neural networks. In           the first part          we will investigate the usefulness, or lack thereof, of influence functions for the interpretation of a classifier's outputs.    </p> <p>     For our study we choose a pre-trained ResNet18, fine-tuned on the           tiny-imagenet dataset          . This dataset was created for a Stanford course on           Deep Learning for Computer Vision          , and is a subset of the           famous ImageNet          with 200 classes instead of 1000, and images down-sampled to a lower resolution of 64x64 pixels.    </p> <p>     After tuning the last layers of the network, we will use           pyDVL          to find the most and the least influential training images for the test set. This can sometimes be used to explain inference errors, or to direct efforts during data collection, although we will face inconclusive results with our model and data. This illustrates well-known issues of influence functions for neural networks.    </p> <p>     However, in the           final part of the notebook          we will see that influence functions are an effective tool for finding anomalous or corrupted data points.    </p> <p>     We conclude with an           appendix          with some basic theoretical concepts used.    </p>"},{"location":"examples/influence_imagenet/#imports-and-setup","title":"Imports and setup","text":""},{"location":"examples/influence_imagenet/#loading-and-preprocessing-the-dataset","title":"Loading and preprocessing the dataset","text":"<p>     We pick two classes arbitrarily to work with: 90 and 100, corresponding respectively to dining tables, and boats in Venice (you can of course select any other two classes, or more of them, although that would imply longer training times and some modifications in the notebook below). The dataset is loaded with     <code>      load_preprocess_imagenet()     </code>     , which returns three pandas     <code>      DataFrames     </code>     with training, validation and test sets respectively. Each dataframe has three columns: normalized images, labels and the original images. Note that you can load a subset of the data decreasing downsampling_ratio.    </p>"},{"location":"examples/influence_imagenet/#model-definition-and-training","title":"Model definition and training","text":"<p>     We use a ResNet18 from     <code>      torchvision     </code>     with final layers modified for binary classification.    </p> <p>     For training, we use the convenience class     <code>      TrainingManager     </code>     which transparently handles persistence after training. It is not part of the main pyDVL package but just a way to reduce clutter in this notebook.    </p> <p>     We train the model for 50 epochs and save the results. Then we plot the train and validation loss curves.    </p>"},{"location":"examples/influence_imagenet/#influence-computation","title":"Influence computation","text":"<p>     Let's now calculate influences! The central interface for computing influences is           InfluenceFunctionModel          . Since Resnet18 is quite big, we pick the conjugate gradient implementation           CgInfluence          , which takes a trained           torch.nn.Module          , the training loss and the training data. Other important parameters are the Hessian regularization term, which should be chosen as small as possible for the computation to converge (further details on why this is important can be found in the           Appendix          ).    </p>"},{"location":"examples/influence_imagenet/#analysing-influences","title":"Analysing influences","text":"<p>     With the computed influences we can study single images or all of them together:    </p>"},{"location":"examples/influence_imagenet/#influence-on-a-single-test-image","title":"Influence on a single test image","text":"<p>     Let's take any image in the test set:    </p>"},{"location":"examples/influence_imagenet/#analysing-the-average-influence-on-test-samples","title":"Analysing the average influence on test samples","text":"<p>     By averaging across the rows of the influence matrix, we obtain the average influence of each training sample on the whole test set:    </p>"},{"location":"examples/influence_imagenet/#detecting-corrupted-data","title":"Detecting corrupted data","text":"<p>     After facing the shortcomings of influence functions for explaining decisions, we move to an application with clear-cut results. Influences can be successfully used to detect corrupted or mislabeled samples, making them an effective tool to \"debug\" training data.    </p> <p>     We begin by training a new model (with the same architecture as before) on a dataset with some corrupted labels. The method     <code>      get_corrupted_imagenet     </code>     will take the training dataset and corrupt a certain fraction of the labels by flipping them. We use the same number of epochs and optimizer as before.    </p>"},{"location":"examples/influence_imagenet/#theory-of-influence-functions-for-neural-networks","title":"Theory of influence functions for neural networks","text":"<p>     In this appendix we will briefly go through the basic ideas of influence functions adapted for neural networks as introduced in           Koh, Pang Wei, and Percy Liang.             \"Understanding Black-box Predictions via Influence Functions\"            International conference on machine learning. PMLR, 2017.      </p> <p>     Note however that this paper departs from the standard and established theory and notation for influence functions. For a rigorous introduction to the topic we recommend classical texts like           Hampel, Frank R., Elvezio M. Ronchetti, Peter J. Rousseeuw, and Werner A. Stahel. Robust Statistics: The Approach Based on Influence Functions. 1st edition. Wiley Series in Probability and Statistics. New York: Wiley-Interscience, 2005. https://doi.org/10.1002/9781118186435.      </p>"},{"location":"examples/influence_imagenet/#upweighting-points","title":"Upweighting points","text":"<p>     Let's start by considering some input space           \\(\\mathcal{X}\\)          to a model (e.g. images) and an output space           \\(\\mathcal{Y}\\)          (e.g. labels). Let's take           \\(z_i = (x_i, y_i)\\)          to be the           \\(i\\)          -th training point, and           \\(\\theta\\)          to be the (potentially highly) multi-dimensional parameters of the neural network (i.e.           \\(\\theta\\)          is a big array with very many parameters). We will indicate with           \\(L(z, \\theta)\\)          the loss of the model for point           \\(z\\)          and parameters           \\(\\theta\\)          . When training the model we minimize the loss over all points, i.e. the optimal parameters are calculated through gradient descent on the following formula:    </p>      \\[ \\hat{\\theta} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) \\]     <p>     where           \\(n\\)          is the total number of training data points.    </p> <p>     For notational  convenience, let's define    </p>      \\[ \\hat{\\theta}_{-z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{z_i \\ne z} L(z_i, \\theta) \\ , \\]     <p>     i.e.           \\(\\hat{\\theta}_{-z}\\)          are the model parameters that minimize the total loss when           \\(z\\)          is not in the training dataset.    </p> <p>     In order to check the impact of each training point on the model, we would need to calculate           \\(\\hat{\\theta}_{-z}\\)          for each           \\(z\\)          in the training dataset, thus re-training the model at least ~           \\(n\\)          times (more if model training is noisy). This is computationally very expensive, especially for big neural networks. To circumvent this problem, we can just calculate a first order approximation of           \\(\\hat{\\theta}\\)          . This can be done through single backpropagation and without re-training the full model.    </p> <p>     Let's define    </p>      \\[ \\hat{\\theta}_{\\epsilon, z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) + \\epsilon L(z, \\theta) \\ , \\]     <p>     which is the optimal           \\(\\hat{\\theta}\\)          if we were to up-weigh           \\(z\\)          by an amount           \\(\\epsilon\\)          .    </p> <p>     From a classical result (a simple derivation is available in Appendix A of Koh and Liang's paper), we know that:    </p>      \\[ \\frac{d \\ \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta L(z, \\hat{\\theta}) \\]     <p>     where           \\(H_{\\hat{\\theta}} = \\frac{1}{n} \\sum_{i=1}^n \\nabla_\\theta^2 L(z_i, \\hat{\\theta})\\)          is the Hessian of           \\(L\\)          . Importantly, notice that this expression is only valid when           \\(\\hat{\\theta}\\)          is a minimum of           \\(L\\)          , or otherwise           \\(H_{\\hat{\\theta}}\\)          cannot be inverted!    </p>"},{"location":"examples/influence_imagenet/#approximating-the-influence-of-a-point","title":"Approximating the influence of a point","text":"<p>     We will define the influence of training point           \\(z\\)          on test point           \\(z_{\\text{test}}\\)          as           \\(\\mathcal{I}(z, z_{\\text{test}}) =  L(z_{\\text{test}}, \\hat{\\theta}_{-z}) - L(z_{\\text{test}}, \\hat{\\theta})\\)          (notice that it is higher for points           \\(z\\)          which positively impact the model score, since if they are excluded, the loss is higher). In practice, however, we will always use the infinitesimal approximation           \\(\\mathcal{I}_{up}(z, z_{\\text{test}})\\)          , defined as    </p>      \\[  \\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\frac{d L(z_{\\text{test}}, \\hat{\\theta}_{\\epsilon, z})}{d \\epsilon} \\Big|_{\\epsilon=0} \\]     <p>     Using the chain rule and the results calculated above, we thus have:    </p>      \\[  \\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_\\theta L(z, \\hat{\\theta}) \\]     <p>     In order to calculate this expression we need the gradient and the Hessian of the loss wrt. the model parameters           \\(\\hat{\\theta}\\)          . This can be easily done through a single backpropagation pass.    </p>"},{"location":"examples/influence_imagenet/#regularizing-the-hessian","title":"Regularizing the Hessian","text":"<p>     One very important assumption that we make when approximating influence is that           \\(\\hat{\\theta}\\)          is at least a local minimum of the loss. However, we clearly cannot guarantee this except for convex models, and despite good apparent convergence,           \\(\\hat{\\theta}\\)          might be located in a region with flat curvature or close to a saddle point. In particular, the Hessian might have vanishing eigenvalues making its direct inversion impossible.    </p> <p>     To circumvent this problem, instead of inverting the true Hessian           \\(H_{\\hat{\\theta}}\\)          , one can invert a small perturbation thereof:           \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\)          , with           \\(\\mathbb{I}\\)          being the identity matrix. This standard trick ensures that the eigenvalues of           \\(H_{\\hat{\\theta}}\\)          are bounded away from zero and therefore the matrix is invertible. In order for this regularization not to corrupt the outcome too much, the parameter           \\(\\lambda\\)          should be as small as possible while still allowing a reliable inversion of           \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\)          .    </p>"},{"location":"examples/influence_sentiment_analysis/","title":"For language models","text":"<p>     This notebooks showcases the use of influence functions for large language models. In particular, it focuses on sentiment analysis using the           IMDB dataset          and a fine-tuned           BERT          model.    </p> <p>     Not all the methods for influence function calculation can scale to large models and datasets. In this notebook we will use the           Kronecker-Factored Approximate Curvature          method, which is the only one that can scale to current state-of-the-art language models.    </p> <p>     The notebook is structured as follows:    </p> <ul> <li>        Setup            imports the required libraries and downloads the dataset and the model.     </li> <li>        Sentiment analysis            loads the model and the dataset and goes through a few examples of sentiment analysis.     </li> <li>        Model and data preparation            prepares the model and the dataset for influence function calculation. In particular, it assigns all the linear layers to require gradients and wraps the model so that only logits are returned (and not the loss or attention masks).     </li> <li>        Influence function computation            : shows how to calculate the influence function for a few test and train examples.     </li> <li>        Analysis of influence values            : analyses the influence values, trying to extract general information about the model and how it is affected by corruption in the training data.     </li> <li>        Influence functions by layer            : since ekfac is based on a block diagonal approximation of the Fisher information matrix, we can compute the influence function separately for each layer of the neural network. This section shows how to do that and how to analyse the results.     </li> </ul> <p>     Finally, the           Appendix          shows how to select the Hessian regularization parameter to obtain the best influence function approximation.    </p>      If you are reading this in the documentation, some boilerplate has been omitted for convenience.     <p>     Let's start by importing the required libraries. If not already installed, you can install them with     <code>      pip install -r requirements-notebooks.txt     </code>     .    </p> <pre><code>import os\nfrom copy import deepcopy\nfrom typing import Sequence\n\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn.functional as F\nfrom datasets import load_dataset\nfrom IPython.display import HTML, display\nfrom sklearn.metrics import f1_score\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer\n\nfrom pydvl.influence.torch import EkfacInfluence\nfrom support.torch import ImdbDataset, ModelLogitsWrapper\n</code></pre> <p>     Sentiment analysis is the task of classifying a sentence as having a positive or negative sentiment. For example, the sentence \"I love this movie\" has a positive sentiment, while \"I hate this movie\" has a negative sentiment. In this notebook we will use the IMDB dataset, which contains 50,000 movie reviews with corresponding labels. The dataset is split into 25,000 reviews for training and 25,000 reviews for testing. The dataset is balanced, meaning that there are the same number of positive and negative reviews in the training and test set.    </p> <pre><code>imdb = load_dataset(\"imdb\")\n</code></pre> <p>     Let's print an example of review and its label    </p> <pre><code>sample_review = imdb[\"train\"].select([24])\n\nprint(f\"Here is a sample review with label {sample_review['label'][0]}: \\n\")\n\ndisplay(HTML(sample_review[\"text\"][0].split(\"&lt;br/&gt;\")[0]))\ndisplay(HTML(sample_review[\"text\"][0].split(\"&lt;br/&gt;\")[-1]))\n</code></pre> <pre>\n<code>Here is a sample review with label 0: \n\n</code>\n</pre>       Without wishing to be a killjoy, Brad Sykes is responsible for at least two of the most dull and clich\u00e9d films i've ever seen - this being one of them, and Camp Blood being another.            I bought this for \u00a31, but remember, you can't put a price on 71 minutes of your life. You'd do well to avoid this turkey, even at a bargain basement price.      <p>     The review is negative, and so label 0 is associated to negative sentiment.    </p> <p>     The model is a BERT model fine-tuned on the IMDB dataset. BERT is a large language model that has been pre-trained on a large corpus of text. The model was fine-tuned on the IMDB dataset by AssemblyAI and is available on the HuggingFace model hub. We also load its tokenizer, which is used to convert sentences into numeric tokens.    </p> <pre><code>tokenizer = AutoTokenizer.from_pretrained(\"assemblyai/distilbert-base-uncased-sst2\")\nmodel = AutoModelForSequenceClassification.from_pretrained(\n    \"assemblyai/distilbert-base-uncased-sst2\"\n)\n</code></pre> <p>     Even if the model is trained on movie reviews, it can be used to classify any sentence as positive or negative. Let's try it on a simple sentence created by us.    </p> <pre><code>example_phrase = (\n    \"Pydvl is the best data valuation library, and it is fully open-source!\"\n)\n\ntokenized_example = tokenizer(\n    [example_phrase],\n    return_tensors=\"pt\",\n    truncation=True,\n)\n\nmodel_output = model(\n    input_ids=tokenized_example.input_ids,\n)\n</code></pre> <p>     The model output is a     <code>      SequenceClassificationOutput     </code>     object, which contains the logits and other information.    </p> <pre>\n<code>Model Output:\n SequenceClassifierOutput(loss=None, logits=tensor([[-2.6237,  2.8350]], grad_fn=&lt;AddmmBackward0&gt;), hidden_states=None, attentions=None)\n</code>\n</pre> <p>     For calculating probabilities and for the influence functions we only need the logits. Then the softmax function converts the logits into probabilities.    </p> <pre><code>model_predictions = F.softmax(model_output.logits, dim=1)\n</code></pre> <p>     The model is quite confident that the sentence has a positive sentiment, which is correct.    </p> <pre>\n<code>Positive probability: 99.6%\nNegative probability: 0.4%\n</code>\n</pre> <p>     Let's examine the model's f1 score on a small subset of the test set.    </p> <pre><code>sample_test_set = imdb[\"test\"].shuffle(seed=seed).select(range(50 if not is_CI else 5))\nsample_test_set = sample_test_set.map(\n    lambda example: tokenizer(example[\"text\"], truncation=True, padding=\"max_length\"),\n    batched=True,\n)\nsample_test_set.set_format(\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])\nmodel.eval()\nwith torch.no_grad():\n    logits = model(\n        input_ids=sample_test_set[\"input_ids\"],\n        attention_mask=sample_test_set[\"attention_mask\"],\n    ).logits\n    predictions = torch.argmax(logits, dim=1)\n</code></pre> <pre><code>f1_score_value = f1_score(sample_test_set[\"label\"], predictions)\nprint(f\"F1 Score: {round(f1_score_value, 3)}\")\n</code></pre> <pre>\n<code>F1 Score: 0.955\n</code>\n</pre> <p>     In this section we will define two helper function and classes that will be used in the rest of the notebook.    </p> <pre><code>def print_sentiment_preds(\n    model: ModelLogitsWrapper, model_input: torch.Tensor, true_label: int\n):\n    \"\"\"\n    Prints the sentiment predictions in a human-readable format given a model and an\n    input. It also prints the true label.\n    \"\"\"\n    model_predictions = F.softmax(model(model_input.unsqueeze(0)), dim=1)\n    print(\n        \"Positive probability: \"\n        + str(round(model_predictions[0][1].item(), 3) * 100)\n        + \"%\"\n    )\n    print(\n        \"Negative probability: \"\n        + str(round(model_predictions[0][0].item(), 3) * 100)\n        + \"%\"\n    )\n\n    true_label = \"Positive\" if true_label == 1 else \"Negative\"\n    print(f\"True label: {true_label} \\n\")\n\n\ndef strip_layer_names(param_names: Sequence[str]):\n    \"\"\"\n    Helper function that strips the parameter names of the model and the transformer,\n    so that they can be printed and compared more easily.\n    \"\"\"\n    stripped_param_names = []\n    for name in param_names:\n        name = name.replace(\"model.\", \"\")\n        if name.startswith(\"distilbert.transformer.\"):\n            name = name.replace(\"distilbert.transformer.\", \"\")\n        stripped_param_names.append(name)\n    return stripped_param_names\n</code></pre> <p>     Importantly, we will need to assign all the linear layers to require gradients, so that we can compute the influence function with respect to them. Keep in mind that the current implementation of Ekfac only supports linear layers, so if any other type of layer in the model requires gradients the initialisation of the influence function class will fail.    </p> <pre><code>for param in model.named_parameters():\n    param[1].requires_grad = False\n\nfor m_name, module in model.named_modules():\n    if len(list(module.children())) == 0 and len(list(module.parameters())) &amp;gt; 0:\n        if isinstance(module, torch.nn.Linear):\n            for p_name, param in module.named_parameters():\n                if (\n                    (\"ffn\" in m_name and not is_CI)\n                    or \"pre_classifier\" in m_name\n                    or \"classifier\" in m_name\n                ):\n                    param.requires_grad = True\n</code></pre> <p>     Albeit restrictive, linear layers constitute a large fraction of the parameters of most large language models, and so our analysis still holds a lot of information about the full neural network.    </p> <pre>\n<code>Total parameters: 66.96 millions\nParameters requiring gradients: 28.93 millions\nRatio of Linear over other layer types: 43.20%\n</code>\n</pre> <p>     We are now ready to compute the influence function for a few testing and training examples. Let's start by selecting a subset of the full training and testing dataset and wrapping them in a     <code>      DataLoader     </code>     object, so that we can easily do batching.    </p> <pre><code>NUM_TRAIN_EXAMPLES = 100 if not is_CI else 7\nNUM_TEST_EXAMPLES = 100 if not is_CI else 5\n\nsmall_train_dataset = (\n    imdb[\"train\"]\n    .shuffle(seed=seed)\n    .select([i for i in list(range(NUM_TRAIN_EXAMPLES))])\n)\nsmall_test_dataset = (\n    imdb[\"test\"].shuffle(seed=seed).select([i for i in list(range(NUM_TEST_EXAMPLES))])\n)\n\ntrain_dataset = ImdbDataset(small_train_dataset, tokenizer=tokenizer)\ntest_dataset = ImdbDataset(small_test_dataset, tokenizer=tokenizer)\n\ntrain_dataloader = torch.utils.data.DataLoader(\n    train_dataset, batch_size=7, shuffle=True\n)\ntest_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=5, shuffle=True)\n</code></pre> <p>     For influence computation we need to take the model in evaluation mode, so that no dropout or batch normalization is applied. Then, we can fit the Ekfac representation.    </p> <pre><code>wrapped_model = ModelLogitsWrapper(model)\nwrapped_model.eval()\n\nekfac_influence_model = EkfacInfluence(\n    wrapped_model,\n    progress=True,\n)\nekfac_influence_model = ekfac_influence_model.fit(train_dataloader)\n</code></pre> <pre>\n<code>K-FAC blocks - batch progress:   0%|          | 0/15 [00:00&lt;?, ?it/s]</code>\n</pre> <p>     And the approximate Hessian is thus obtained. Considering that the model has almost 30 million parameters requiring gradients, this was very fast! Of course, this Hessian is computed using only a very small fraction (~0.4%) of the training data, and for a better approximation we should use a larger subset.    </p> <p>     Before continuing, we need to set the Hessian regularization parameter to an appropriate value. A way to decide which is better can be found in the           Appendix          . Here, we will just set it to 1e-5.    </p> <pre><code>ekfac_influence_model.hessian_regularization = 1e-5\n</code></pre> <p>     We calculate the influence of the first batch of training data over the first batch of test data. This is because influence functions are very expensive to compute, and so to keep the runtime of this notebook within a few minutes we need to restrict ourselves to a small number of examples.    </p> <pre><code>test_input, test_labels, test_text = next(iter(test_dataloader))\ntrain_input, train_labels, train_text = next(iter(train_dataloader))\n</code></pre> <p>     And let's finally compute the influence function values    </p> <pre><code>ekfac_train_influences = ekfac_influence_model.influences(\n    test_input,\n    test_labels,\n    train_input,\n    train_labels,\n)\n</code></pre> <pre>\n<code>/home/jakob/Documents/pyDVL/venv/lib/python3.10/site-packages/transformers/models/distilbert/modeling_distilbert.py:222: UserWarning: There is a performance drop because we have not yet implemented the batching rule for aten::masked_fill.Tensor. Please file us an issue on GitHub so that we can prioritize its implementation. (Triggered internally at ../aten/src/ATen/functorch/BatchedFallback.cpp:82.)\n  scores = scores.masked_fill(\n</code>\n</pre> <p>     Now that we have calculated the influences for a few examples, let's analyse some of the extreme values.    </p> <p>     Let's plot the influence values as a heatmap for easily spotting patterns.    </p> <p>     Most of the test and training examples have similar influence, close to zero. However, there is one test and one training samples that stand out. In particular, their cross influence is very large and negative. Let's examine them more closely.    </p> <pre>\n<code>Training example with idx 3: \n\nPositive probability: 18.099999999999998%\nNegative probability: 81.89999999999999%\nTrue label: Positive \n\nSentence:\n</code>\n</pre>       In the process of trying to establish the audiences' empathy with Jake Roedel (Tobey Maguire) the filmmakers slander the North and the Jayhawkers. Missouri never withdrew from the Union and the Union Army was not an invading force. The Southerners fought for State's Rights: the right to own slaves, elect crooked legislatures and judges, and employ a political spoils system. There's nothing noble in that. The Missourians could have easily traveled east and joined the Confederate Army.             It seems to me that the story has nothing to do with ambiguity. When Jake leaves the Bushwhackers, it's not because he saw error in his way, he certainly doesn't give himself over to the virtue of the cause of abolition.      <p>     We can see that, despite being positive, this review is quite hard to classify. Its language is overall negative, mostly associated to the facts narrated rather than the movie itself. Notice how several terms are related to war and invasion.    </p> <pre>\n<code>Test example with idx 4: \n\nPositive probability: 39.6%\nNegative probability: 60.4%\nTrue label: Negative \n\nSentence:\n</code>\n</pre>       \"An astronaut (Michael Emmet) dies while returning from a mission and his body is recovered by the military. The base where the dead astronaut is taken to becomes the scene of a bizarre invasion plan from outer space. Alien embryos inside the dead astronaut resurrect the corpse and begin a terrifying assault on the military staff in the hopes of conquering the world,\" according to the DVD sleeve's synopsis.             A Roger Corman \"American International\" production. The man who fell to Earth impregnated, Mr. Emmet (as John Corcoran), does all right. Angela Greene is his pretty conflicted fianc\u00e9e. And, Ed Nelson (as Dave Randall) is featured as prominently. With a bigger budget, better opening, and a re-write for crisper characterizations, this could have been something approaching classic 1950s science fiction.             *** Night of the Blood Beast (1958) Bernard L. Kowalski, Roger Corman ~ Michael Emmet, Angela Greene, Ed Nelson      <p>     This review is also quite hard to classify. This time it has a negative sentiment towards the movie, but it also contains several words with positive connotation. The parallel with the previous review is quite interesting since both talk about an invasion.    </p> <p>     As it is often the case when analysing influence functions, it is hard to understand why these examples have such a large influence. We have seen some interesting patterns, mostly related to similarities in the language and words used, but it is hard to say with certainty if these are the reasons for such a large influence.    </p> <p>     A           recent paper          has explored this topic in high detail, even for much larger language models than BERT (up to ~50 billion parameters!). Among the most interesting findings is that smaller models tend to rely a lot on word-to-word correspondencies, while larger models are more capable of extracting higher level concepts, drawing connections between words across multiple phrases.    </p> <p>     For more info, you can visit our           blog on influence functions for large language models      </p> <p>     In this sections we want to get an idea of how influence functions change when training examples are corrupted. In the next cell we will flip the label of all the training examples and compute the influences on the same test batch as before.    </p> <pre><code>modified_train_labels = deepcopy(train_labels)\nmodified_train_labels = 1 - train_labels\n\ncorrupted_ekfac_train_influences = ekfac_influence_model.influences(\n    test_input,\n    test_labels,\n    train_input,\n    modified_train_labels,\n)\n</code></pre> <p>     Overall, when corrupted the influences tend to become negative, as expected. Nevertheless, there are cases where values go from slightly negative to positive, mostly isolated to the second and last test samples. Single values can be quite noisy, so it is difficult to generalise this result, but it would be interesting to see how common these cases are in the full test dataset.    </p> <p>     Since ekfac is based on a block diagonal approximation of the Fisher information matrix, we can compute the influence functions separately for each layer of the neural network. In this section we show how to do that and we briefly analyse the results.    </p> <pre><code>influences_by_layer = ekfac_influence_model.influences_by_layer(\n    test_input,\n    test_labels,\n    train_input,\n    train_labels,\n)\n</code></pre> <p>     The method     <code>      influences_by_layer     </code>     returns a dictionary containing the influence function values for each layer of the neural network as a tensor. To recover the full influence values as returned by the     <code>      influences     </code>     (as done in the previous section), we need to sum each layer's values.    </p> <pre><code>influences = torch.zeros_like(ekfac_train_influences)\nfor layer_id, value in influences_by_layer.items():\n    influences += value.detach()\n</code></pre> <p>     And if we plot the result as a heatmap we can see that the results are the same as in           Negative influence training examples      </p> <p>     Let's analyse how the influence values change across different layers for given test and train examples.    </p> <p>     The plot above shows the influences for test idx 0 and all train idx apart idx=3 (excluded for clarity since it has a very large absolute value). We can see that the scores tend to keep their sign across layers, but in almost all cases tend to decrease when approaching the output layer. This is not always the case, and in fact other test examples show different patterns. Understanding why this happens is an interesting research direction.    </p> <p>     Ekfac is a powerful approximate method for computing the influence function of models that use a cross-entropy loss. In this notebook we applied it to sentiment analysis with BERT on the IMDB dataset. However, this method can be applied to much larger models and problems, e.g. to analyse the influence of entire sentences generated by GPT, Llama or Claude. For more info, you can visit our           paper pill on influence functions for large language models      </p> <p>     The Hessian regularization value impacts a lot the quality of the influence function approximation. In general, the value should be chosen as small as possible so that the results are finite. In practice, even when finite the influence values can be too large and lead to numerical instabilities. In this section we show how to efficiently analyse the impact of the Hessian regularization value with the ekfac method.    </p> <p>     Let's start with a few additional imports.    </p> <pre><code>import pandas as pd\nfrom scipy.stats import pearsonr, spearmanr\n</code></pre> <p>     The method     <code>      explore_hessian_regularization     </code>     will calculate the influence values of the training examples with each other for a range of Hessian regularization values. The method optimises gradient calculation and Hessian inversion to minimise the computation time.    </p> <pre><code>influences_by_reg_value = ekfac_influence_model.explore_hessian_regularization(\n    train_input,\n    train_labels,\n    regularization_values=[1e-15, 1e-9, 1e-5, 1],\n)\n</code></pre> <pre>\n<code>/home/jakob/Documents/pyDVL/venv/lib/python3.10/site-packages/transformers/models/distilbert/modeling_distilbert.py:222: UserWarning: There is a performance drop because we have not yet implemented the batching rule for aten::masked_fill.Tensor. Please file us an issue on GitHub so that we can prioritize its implementation. (Triggered internally at ../aten/src/ATen/functorch/BatchedFallback.cpp:82.)\n  scores = scores.masked_fill(\n</code>\n</pre> <p>     The resulting object,     <code>      influences_by_reg_value     </code>     is a dictionary that associates to each regularization value the influences for each layer of the neural network. This is a lot of data, so we will first organise it in a pandas dataframe and take the average across training examples.    </p> <pre><code>cols = [\"reg_value\", \"layer_id\", \"mean_infl\"]\ninfl_df = pd.DataFrame(influences_by_reg_value, columns=cols)\nfor reg_value in influences_by_reg_value:\n    for layer_id, layer_influences in influences_by_reg_value[reg_value].items():\n        mean_infl = torch.mean(layer_influences, dim=0).detach().numpy()\n        infl_df = pd.concat(\n            [infl_df, pd.DataFrame([[reg_value, layer_id, mean_infl]], columns=cols)]\n        )\n</code></pre> <pre>\n<code>/tmp/ipykernel_8503/1081261490.py:6: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n  infl_df = pd.concat(\n</code>\n</pre> <p>     With this dataframe, we can take contiguous values of regularization and, for each layer, calculate the Pearson and Spearman correlation coefficients. This will give us an idea of how the influence values change with the regularization value.    </p> <pre><code>result_corr = {}\nfor layer_id, group_df in infl_df.groupby(\"layer_id\"):\n    result_corr[layer_id + \"_pearson\"] = {}\n    result_corr[layer_id + \"_spearman\"] = {}\n    for idx, mean_infl in enumerate(group_df[\"mean_infl\"]):\n        if idx == 0:\n            continue\n        reg_value_diff = f\"Reg: {group_df['reg_value'].iloc[idx-1]} -&amp;gt; {group_df['reg_value'].iloc[idx]}\"\n        pearson = pearsonr(mean_infl, group_df[\"mean_infl\"].iloc[idx - 1]).statistic\n        spearman = spearmanr(mean_infl, group_df[\"mean_infl\"].iloc[idx - 1]).statistic\n        result_corr[layer_id + \"_pearson\"].update({f\"{reg_value_diff}\": pearson})\n        result_corr[layer_id + \"_spearman\"].update({f\"{reg_value_diff}\": spearman})\nresult_df = pd.DataFrame(result_corr).T\n</code></pre> <p>     Let's plot the correlations heatmap. The y-axis reports Spearman and Pearson correlations for each layer, while the x-axis reports pairs of regularization values. High correlations mean that influences are stable across regularization values.    </p> <p>     In our case, we can see that for regularization = 1 the spearman correlation becomes very bad. However, for a large range of regularization values smaller than 1 the sample rankings are stable. This is a good indicator that the model is not too sensitive to the regularization value. We therefore chose the value 1e-5 for our analysis.    </p>"},{"location":"examples/influence_sentiment_analysis/#influence-functions-for-large-language-models","title":"Influence functions for Large Language Models","text":""},{"location":"examples/influence_sentiment_analysis/#setup","title":"Setup","text":""},{"location":"examples/influence_sentiment_analysis/#sentiment-analysis","title":"Sentiment Analysis","text":""},{"location":"examples/influence_sentiment_analysis/#model-and-data-preparation","title":"Model and Data Preparation","text":""},{"location":"examples/influence_sentiment_analysis/#influence-function-computation","title":"Influence function computation","text":""},{"location":"examples/influence_sentiment_analysis/#analysis-of-influence-values","title":"Analysis of influence values","text":""},{"location":"examples/influence_sentiment_analysis/#negative-influence-training-examples","title":"Negative influence training examples","text":""},{"location":"examples/influence_sentiment_analysis/#influence-of-corrupted-training-examples","title":"Influence of corrupted training examples","text":""},{"location":"examples/influence_sentiment_analysis/#influence-functions-by-layer","title":"Influence functions by layer","text":""},{"location":"examples/influence_sentiment_analysis/#conclusion","title":"Conclusion","text":""},{"location":"examples/influence_sentiment_analysis/#appendix-choosing-the-hessian-regularization-value","title":"Appendix: Choosing the Hessian regularization value","text":""},{"location":"examples/influence_synthetic/","title":"For mislabeled data","text":"If you are reading this in the documentation, some boilerplate has been omitted for convenience.     <pre><code>%autoreload\n%matplotlib inline\n\nimport os\nimport random\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nfrom pydvl.influence.torch import DirectInfluence, CgInfluence\nfrom support.shapley import (\n    synthetic_classification_dataset,\n    decision_boundary_fixed_variance_2d,\n)\nfrom support.common import (\n    plot_gaussian_blobs,\n    plot_losses,\n    plot_influences,\n)\nfrom support.torch import (\n    fit_torch_model,\n    TorchLogisticRegression,\n)\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\nfrom torch.optim import AdamW, lr_scheduler\nfrom torch.utils.data import DataLoader, TensorDataset\n</code></pre> <p>     The following code snippet generates the aforementioned dataset.    </p> <pre><code>train_data, val_data, test_data = synthetic_classification_dataset(\n    means, sigma, num_samples, train_size=0.7, test_size=0.2\n)\n</code></pre> <p>     Given the simplicity of the dataset, we can calculate exactly the optimal decision boundary(that which maximizes our accuracy). The following code maps a continuous line of z values to a 2-dimensional vector in feature space (More details are in the appendix to this notebook.)    </p> <pre><code>decision_boundary_fn = decision_boundary_fixed_variance_2d(means[0], means[1])\ndecision_boundary = decision_boundary_fn(np.linspace(-1.5, 1.5, 100))\n</code></pre> <pre><code>plot_gaussian_blobs(\n    train_data,\n    test_data,\n    xlabel=\"$x_0$\",\n    ylabel=\"$x_1$\",\n    legend_title=\"$y - labels$\",\n    line=decision_boundary,\n    s=10,\n    suptitle=\"Plot of train-test data\",\n)\n</code></pre> <p>     Note that there are samples which go across the optimal decision boundary and will be wrongly labelled. The optimal decision boundary can not discriminate these as the mislabelling is a consequence of the presence of random noise.    </p> <pre><code>model = TorchLogisticRegression(num_features)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\nnum_epochs = 50\nlr = 0.05\nweight_decay = 0.05\nbatch_size = 256\n\ntrain_data_loader = DataLoader(\n    TensorDataset(\n        torch.as_tensor(train_data[0]),\n        torch.as_tensor(train_data[1], dtype=torch.float64).unsqueeze(-1),\n    ),\n    batch_size=batch_size,\n    shuffle=True,\n)\n\nval_data_loader = DataLoader(\n    TensorDataset(\n        torch.as_tensor(val_data[0]),\n        torch.as_tensor(val_data[1], dtype=torch.float64).unsqueeze(-1),\n    ),\n    batch_size=batch_size,\n    shuffle=True,\n)\n\noptimizer = AdamW(params=model.parameters(), lr=lr, weight_decay=weight_decay)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\nlosses = fit_torch_model(\n    model=model,\n    training_data=train_data_loader,\n    val_data=val_data_loader,\n    loss=F.binary_cross_entropy,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    num_epochs=num_epochs,\n    device=device,\n)\n</code></pre> <p>     And let's check that the model is not overfitting    </p> <pre><code>plot_losses(losses)\n</code></pre> <p>     A look at the confusion matrix also shows good results    </p> <p>     It is important that the model converges to a point near the optimum, since the influence values assume that we are at a minimum (or close) in the loss landscape. The function    </p>      \\[I(x_1, y_1, x_2, y_2) \\colon \\mathbb{R}^d \\times \\mathbb{R}^d \\to \\mathbb{R}\\]     <p>     measures the influence of the data point           \\(x_1\\)          onto           \\(x_2\\)          conditioned on the training targets           \\(y_1\\)          and           \\(y_2\\)          trough some model parameters           \\(\\theta\\)          . If the loss function L is differentiable, we can take           \\(I\\)          to be    </p> <p>     $$ I(x_1, x_2) = \\nabla_\\theta\\; L(x_1, y_1) ^\\mathsf{T} \\; H_\\theta^{-1} \\; \\nabla_\\theta \\; L(x_2, y_2) $$ See           \"Understanding Black-box Predictions via Influence Functions\"          for a detailed derivation of this formula    </p> <p>     Let's take a subset of the training data points, which we will calculate the influence values of.    </p> <pre><code>x = train_data[0][:100]\ny = train_data[1][:100]\n</code></pre> <p>     In pyDVL, the influence of the training points on the test points can be calculated with the following    </p> <pre><code>train_x = torch.as_tensor(x)\ntrain_y = torch.as_tensor(y, dtype=torch.float64).unsqueeze(-1)\ntest_x = torch.as_tensor(test_data[0])\ntest_y = torch.as_tensor(test_data[1], dtype=torch.float64).unsqueeze(-1)\n\ntrain_data_loader = DataLoader(\n    TensorDataset(train_x, train_y),\n    batch_size=batch_size,\n)\n\ninfluence_model = DirectInfluence(\n    model,\n    F.binary_cross_entropy,\n    hessian_regularization=0.0,\n)\ninfluence_model = influence_model.fit(train_data_loader)\n\ninfluence_values = influence_model.influences(\n    test_x, test_y, train_x, train_y, mode=\"up\"\n)\n</code></pre> <p>     The above explicitly constructs the Hessian. This can often be computationally expensive and conjugate gradient approximate calculation should be used for bigger models.    </p> <p>     With the influence type 'up', training influences have shape [NxM] where N is the number of test samples and M is the number of training samples. They therefore associate to each training sample its influence on each test sample.  Influence type 'perturbation', instead, return an array of shape  [NxMxF], where F is the number of features in input, ie. the length of x.    </p> <p>     In our case, in order to have a value of the total average influence of a point we can just average across training samples.    </p> <pre><code>mean_train_influences = np.mean(influence_values.cpu().numpy(), axis=0)\n</code></pre> <p>     Let's plot the results (adjust colorbar_limits for better color gradient)    </p> <pre><code>plot_influences(\n    x,\n    mean_train_influences,\n    line=decision_boundary,\n    xlabel=\"$x_0$\",\n    ylabel=\"$x_1$\",\n    suptitle=\"Influences of input points\",\n    legend_title=\"influence values\",\n    # colorbar_limits=(-0.3,),\n);\n</code></pre> <p>     We can see that, as we approach the separation line, the influences tend to move away from zero, i.e. the points become more decisive for model training, some in a positive way, some negative.    </p> <p>     As a further test, let's introduce some labelling errors into           \\(y\\)          and see how the distribution of the influences changes. Let's flip the first 10 labels and calculate influences    </p> <pre><code>y_corrupted = np.copy(y)\ny_corrupted[:10] = [1 - yi for yi in y[:10]]\ntrain_y_corrupted = torch.as_tensor(y_corrupted, dtype=torch.float64).unsqueeze(-1)\ntrain_corrupted_data_loader = DataLoader(\n    TensorDataset(\n        train_x,\n        train_y_corrupted,\n    ),\n    batch_size=batch_size,\n)\n\ninfluence_model = DirectInfluence(\n    model,\n    F.binary_cross_entropy,\n    hessian_regularization=0.0,\n)\ninfluence_model = influence_model.fit(train_corrupted_data_loader)\ninfluence_values = influence_model.influences(\n    test_x, test_y, train_x, train_y_corrupted, mode=\"up\"\n)\n\nmean_train_influences = np.mean(influence_values.cpu().numpy(), axis=0)\n</code></pre> <pre>\n<code>Average mislabelled data influence: -0.8450009471117079\nAverage correct data influence: 0.010396852920315886\n</code>\n</pre> <p>     Red circles indicate the points which have been corrupted. We can see that the mislabelled data have a more negative average influence on the model, especially those that are farther away from the decision boundary.    </p> <p>     The \"direct\" method that we have used above involves the inversion of the Hessian matrix of the model. If a model has           \\(n\\)          training points and           \\(\\theta  \\in \\mathbb{R}^p\\)          parameters, this requires           \\(O(n \\ p^2 + p^3)\\)          operations, which for larger models, like neural networks, becomes quickly unfeasible. Conjugate gradient avoids the explicit computation of the Hessian via a technique called implicit Hessian-vector products (HVPs), which typically takes           \\(O(n \\ p)\\)          operations.    </p> <p>     In the next cell we will use conjugate gradient to compute the influence factors. Since logistic regression is a very simple model, \"cg\" actually slows computation with respect to the direct method, which in this case is a much better choice. Nevertheless, we are able to verify that the influences calculated with \"cg\" are the same (to a minor error) as those calculated directly.    </p> <pre><code>influence_model = CgInfluence(\n    model,\n    F.binary_cross_entropy,\n    hessian_regularization=0.0,\n)\ninfluence_model = influence_model.fit(train_corrupted_data_loader)\ninfluence_values = influence_model.influences(\n    test_x, test_y, train_x, train_y_corrupted\n)\nmean_train_influences = np.mean(influence_values.cpu().numpy(), axis=0)\n\nprint(\"Average mislabelled data influence:\", np.mean(mean_train_influences[:10]))\nprint(\"Average correct data influence:\", np.mean(mean_train_influences[10:]))\n</code></pre> <pre>\n<code>Average mislabelled data influence: -0.8448414156979295\nAverage correct data influence: 0.010395021813591145\n</code>\n</pre> <p>     Averages are very similar to the ones calculated through direct method. Same is true for the plot    </p>"},{"location":"examples/influence_synthetic/#influence-functions-for-data-mislabeling","title":"Influence functions for data mislabeling","text":"<p>     In this notebook, we will take a closer look at the theory of influence functions with the help of a synthetic dataset. Data mislabeling occurs whenever some examples from a usually big dataset are wrongly-labeled. In real-life this happens fairly often, e.g. as a consequence of human error, or noise in the data.    </p> <p>     Let's consider a classification problem with the following notation:    </p>      \\[ \\begin{align*} x_i &amp;\\in \\mathbb{R}^d \\\\ y_i &amp;\\in \\{0, 1\\} \\\\ \\forall i &amp;\\in [ N ] \\end{align*} \\]     <p>     In other words, we have a dataset containing           \\(N\\)          samples, each with label 1 or 0. As typical example you can think of y indicating whether a patient has a disease based on some feature representation           \\(x\\)          .    </p> <p>     Let's now introduce a toy model that will help us delve into the theory and practical utility of influence functions. We will assume that           \\(y\\)          is a Bernoulli binary random variable while the input           \\(x\\)          is d-dimensional Gaussian distribution which depends on the label           \\(y\\)          . More precisely:    </p>      \\[ y_i \\sim \\text{Ber}\\left (0.5 \\right) \\\\ x_i \\sim \\mathcal{N}\\left ((1 - y_i) \\mu_1 + y_i \\mu_2, \\sigma^2 I \\right), \\]     <p>     with fixed means and diagonal covariance. Implementing the sampling scheme in python is straightforward and can be achieved by first sampling           \\(y\\)          and afterward           \\(x\\)          .    </p>"},{"location":"examples/influence_synthetic/#imports","title":"Imports","text":""},{"location":"examples/influence_synthetic/#dataset","title":"Dataset","text":""},{"location":"examples/influence_synthetic/#plotting-the-dataset","title":"Plotting the dataset","text":"<p>     Let's plot the dataset is plotted with their respective labels and the optimal decision line    </p>"},{"location":"examples/influence_synthetic/#training-the-model","title":"Training the model","text":"<p>     We will now train a logistic regression model on the training data. This can be done with the following    </p>"},{"location":"examples/influence_synthetic/#calculating-influences","title":"Calculating influences","text":""},{"location":"examples/influence_synthetic/#inversion-through-conjugate-gradient","title":"Inversion through conjugate gradient","text":""},{"location":"examples/influence_synthetic/#appendix-calculating-the-decision-boundary","title":"Appendix: Calculating the decision boundary","text":"<p>     For obtaining the optimal discriminator one has to solve the equation    </p>      \\[p(x|y=0)=p(x|y=1)\\]     <p>     and determine the solution set           \\(X\\)          . Let's take the following probabilities    </p>      \\[ \\begin{align*} p(x|y=0)&amp;=\\mathcal{N}\\left (\\mu_1, \\sigma^2 I \\right) \\\\ p(x|y=1)&amp;=\\mathcal{N}\\left (\\mu_2, \\sigma^2 I \\right) \\end{align*} \\]     <p>     For a single fixed diagonal variance parameterized by           \\(\\sigma\\)          , the optimal discriminator lays at points which are equidistant from the means of the two distributions, i.e.    </p>      \\[ \\begin{align*} \\| x - \\mu_1 \\|^2 &amp;= \\| x - \\mu_2 \\|^2 \\\\ \\| \\mu_1 \\|^2 -2 x^\\mathsf{T} \\mu_1 &amp;= \\| \\mu_2 \\|^2 -2 x^\\mathsf{T} \\mu_2 \\\\ \\implies 0 &amp;= 2 (\\mu_2 - \\mu_1)^\\mathsf{T} x + \\| \\mu_1 \\|^2 - \\| \\mu_2 \\|^2 \\\\ 0 &amp;= \\mu_1^\\mathsf{T}x - \\mu_2^\\mathsf{T}x - \\frac{1}{2} \\mu_1^\\mathsf{T} \\mu_1 + \\frac{1}{2} \\mu_2^\\mathsf{T} \\mu_2 \\end{align*} \\]     <p>     This is just the implicit description of the line. Solving for the explicit form can be achieved by enforcing a functional form           \\(f(z) = x = a z + b\\)          with           \\(z \\in \\mathbb{R}\\)          onto           \\(x\\)          . After the term is inserted in the previous equation    </p>      \\[ 0 = (\\mu_2 - \\mu_1)^\\mathsf{T} (az + b) + \\frac{1}{2} \\| \\mu_1 \\|^2 - \\| \\mu_2 \\|^2 \\]     <p>     We can write           \\(a\\)          since, by symmetry, it is expected to be explicitly orthogonal to           \\(\\mu_2 - \\mu_1\\)          . Then, solving for           \\(b\\)          , the solution can be found to be    </p>      \\[ f(z) = \\underbrace{\\begin{bmatrix} 0 &amp; 1 \\\\ -1 &amp; 0 \\end{bmatrix} (\\mu_2 - \\mu_1)}_a z + \\underbrace{\\frac{\\mu_1 + \\mu_2}{2}}_b \\]"},{"location":"examples/influence_wine/","title":"For outlier detection","text":"If you are reading this in the documentation, some boilerplate has been omitted for convenience.     <p>     Let's start by loading the imports, the dataset and splitting it into train, validation and test sets. We will use a large test set to have a less noisy estimate of the average influence.    </p> <pre><code>%autoreload\n%matplotlib inline\n\nimport os\nimport random\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nfrom support.common import plot_losses\nfrom support.torch import TorchMLP, fit_torch_model\nfrom pydvl.influence.torch import (\n    DirectInfluence,\n    CgInfluence,\n    ArnoldiInfluence,\n    EkfacInfluence,\n    NystroemSketchInfluence,\n    LissaInfluence,\n)\nfrom support.shapley import load_wine_dataset\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score\nfrom torch.optim import Adam, lr_scheduler\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom scipy.stats import pearsonr, spearmanr\n</code></pre> <pre><code>training_data, val_data, test_data, feature_names = load_wine_dataset(\n    train_size=0.6, test_size=0.3\n)\n</code></pre> <p>     We will corrupt some of the training points by flipping their labels    </p> <pre><code>num_corrupted_idxs = 10\ntraining_data[1][:num_corrupted_idxs] = torch.tensor(\n    [(val + 1) % 3 for val in training_data[1][:num_corrupted_idxs]]\n)\n</code></pre> <p>     and let's wrap it in a pytorch data loader    </p> <pre><code>training_data_loader = DataLoader(\n    TensorDataset(*training_data), batch_size=32, shuffle=False\n)\nval_data_loader = DataLoader(TensorDataset(*val_data), batch_size=32, shuffle=False)\ntest_data_loader = DataLoader(TensorDataset(*test_data), batch_size=32, shuffle=False)\n</code></pre> <pre><code>feature_dimension = 13\nnum_classes = 3\nnetwork_size = [16, 16]\nlayers_size = [feature_dimension, *network_size, num_classes]\nnum_epochs = 300\nlr = 0.005\nweight_decay = 0.01\n\nnn_model = TorchMLP(layers_size)\nnn_model.to(device)\n\noptimizer = Adam(params=nn_model.parameters(), lr=lr, weight_decay=weight_decay)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\n\nlosses = fit_torch_model(\n    model=nn_model,\n    training_data=training_data_loader,\n    val_data=val_data_loader,\n    loss=F.cross_entropy,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    num_epochs=num_epochs,\n    device=device,\n)\n</code></pre> <p>     Let's check that the training has found a stable minimum by plotting the training and validation loss    </p> <pre><code>plot_losses(losses)\n</code></pre> <p>     Since it is a classification problem, let's also take a look at the confusion matrix on the test set    </p> <p>     And let's compute the f1 score of the model    </p> <pre><code>f1_score(test_data[1], pred_y_test, average=\"weighted\")\n</code></pre> <pre>\n<code>0.9633110554163186</code>\n</pre> <p>     Let's now move to calculating influences of each point on the total score.    </p> <pre><code>influence_model = DirectInfluence(\n    nn_model,\n    F.cross_entropy,\n    hessian_regularization=0.1,\n)\ninfluence_model = influence_model.fit(training_data_loader)\ntrain_influences = influence_model.influences(*test_data, *training_data, mode=\"up\")\n</code></pre> <p>     the returned matrix, train_influences, has a quantity of columns equal to the points in the training set, and a number of rows equal to the points in the test set. At each element           \\(a_{i,j}\\)          it stores the influence that training point           \\(j\\)          has on the classification of test point           \\(i\\)          .    </p> <p>     If we take the average across every column of the influences matrix, we obtain an estimate of the overall influence of a training point on the total accuracy of the network.    </p> <pre><code>mean_train_influences = np.mean(train_influences.cpu().numpy(), axis=0)\n</code></pre> <p>     The following histogram shows that there are big differences in score within the training set (notice the log-scale on the y axis).    </p> <p>     We can see that the corrupted points tend to have a negative effect on the model, as expected    </p> <pre>\n<code>Average influence of corrupted points:  -1.0667533\nAverage influence of other points:  0.10814369\n</code>\n</pre> <p>     We have seen how to calculate the influence of single training points on each test point using influence_type 'up'. Using influence_type 'perturbation' we can also calculate the influence of the input features of each point. In the next cell we will calculate the average influence of each feature on training and test points, and ultimately assess which are the most relevant to model performance.    </p> <pre><code>influence_model.hessian_regularization = 1.0\nfeature_influences = influence_model.influences(\n    *test_data, *training_data, mode=\"perturbation\"\n)\n</code></pre> <p>     The explicit calculation of the Hessian matrix is numerically challenging, and due to the high memory need infeasible for larger models.  PyDVL allows to use several approximation methods for the action of the inverse Hessian matrix to overcome this bottleneck:    </p> <ul> <li>      Iteration-based:      <ul> <li>        Conjugate Gradients (Cg)       </li> <li>        Linear time Stochastic Second-Order Approximation (                 LiSSA                )       </li> </ul> </li> <li>      Low-rank Approximations:      <ul> <li>        Arnoldi       </li> <li>        Nystr\u00f6m Sketch-and-Solve (Nystr\u00f6m)       </li> </ul> </li> <li>      Factorization-based:      <ul> <li>        Eigenvalue-corrected Kronecker Factorization (                 EKFAC                )       </li> </ul> </li> </ul> <p>     In the following, we show the usage of these approximation methods and investigate their performance.    </p> <p>     Since the Hessian is symmetric and positive definite (at least after applying a sufficient regularization), we can utilize the           Conjugate Gradients Algorithm          to approximately solve the equations    </p>      \\[ (H + \\lambda \\operatorname{I}) x = b\\]     <p>     Most importantly, the algorithm do not require the computation of the full Hessian matrix, but only requires the implementation of Hessian vector products. pyDVL implements a stable block variant of preconditioned conjugate gradients algorithm.    </p> <pre><code>from pydvl.influence.torch.pre_conditioner import NystroemPreConditioner\n\nnn_model.to(\"cpu\")\ncg_influence_model = CgInfluence(\n    nn_model,\n    F.cross_entropy,\n    hessian_regularization=0.1,\n    progress=True,\n    use_block_cg=True,\n    pre_conditioner=NystroemPreConditioner(rank=5),\n)\ncg_influence_model = cg_influence_model.fit(training_data_loader)\ncg_train_influences = cg_influence_model.influences(\n    *test_data, *training_data, mode=\"up\"\n)\nmean_cg_train_influences = np.mean(cg_train_influences.numpy(), axis=0)\n</code></pre> <p>     Let's compare the results obtained through conjugate gradient with those from the direct method    </p> <pre>\n<code>Percentage error of Cg over direct method:20.877432823181152 %\n</code>\n</pre> <pre>\n<code>Pearson Correlation Cg vs direct 0.9977952163687263\nSpearman Correlation Cg vs direct 0.9972793913897776\n</code>\n</pre> <p>     The           LiSSA          method is a stochastic approximation of the inverse Hessian vector product. Compared to conjugate gradient it is faster but less accurate and typically suffers from instability.    </p> <p>     In order to find the solution of the HVP,           LiSSA          iteratively approximates the inverse of the Hessian matrix with the following update:    </p>      \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\]     <p>     where           \\(d\\)          and           \\(s\\)          are a dampening and a scaling factor.    </p> <pre><code>lissa_influence_model = LissaInfluence(\n    nn_model,\n    F.cross_entropy,\n    hessian_regularization=0.1,\n    progress=True,\n)\nlissa_influence_model = lissa_influence_model.fit(training_data_loader)\nlissa_train_influences = lissa_influence_model.influences(\n    *test_data, *training_data, mode=\"up\"\n)\nmean_lissa_train_influences = np.mean(lissa_train_influences.numpy(), axis=0)\n</code></pre> <pre>\n<code>Percentage error of Lissa over direct method:9.416493028402328 %\n</code>\n</pre> <pre>\n<code>Pearson Correlation Lissa vs direct 0.999796846529674\nSpearman Correlation Lissa vs direct 0.9990729778068873\n</code>\n</pre> <p>     The Arnoldi method leverages a low rank approximation of the Hessian matrix to reduce the memory requirements. It is generally much faster than the conjugate gradient method and can achieve similar accuracy.    </p> <pre><code>arnoldi_influence_model = ArnoldiInfluence(\n    nn_model,\n    F.cross_entropy,\n    rank_estimate=30,\n    hessian_regularization=0.1,\n)\narnoldi_influence_model = arnoldi_influence_model.fit(training_data_loader)\narnoldi_train_influences = arnoldi_influence_model.influences(\n    *test_data, *training_data, mode=\"up\"\n)\nmean_arnoldi_train_influences = np.mean(arnoldi_train_influences.numpy(), axis=0)\n</code></pre> <pre>\n<code>Percentage error of Arnoldi over direct method:37.59587109088898 %\n</code>\n</pre> <pre>\n<code>Pearson Correlation Arnoldi vs direct 0.9873076667742712\nSpearman Correlation Arnoldi vs direct 0.9791621533113334\n</code>\n</pre> <p>     Similar to the Arnoldi method. the Nystr\u00f6m method uses a low-rank approximation, which is computed from random projections of the Hessian matrix. In general the approximation is expected to be worse then the Arnoldi approximation, but is cheaper to compute.    </p> <pre><code>nystroem_influence_model = NystroemSketchInfluence(\n    nn_model,\n    F.cross_entropy,\n    rank=30,\n    hessian_regularization=0.1,\n)\nnystroem_influence_model = nystroem_influence_model.fit(training_data_loader)\nnystroem_train_influences = nystroem_influence_model.influences(\n    *test_data, *training_data, mode=\"up\"\n)\nmean_nystroem_train_influences = np.mean(nystroem_train_influences.numpy(), axis=0)\n</code></pre> <pre>\n<code>Percentage error of Nystr\u00f6m over direct method:37.205782532691956 %\n</code>\n</pre> <pre>\n<code>Pearson Correlation Nystr\u00f6m vs direct 0.9946487475679316\nSpearman Correlation Nystr\u00f6m vs direct 0.985500163740333\n</code>\n</pre> <p>     The           EKFAC          method is a more recent technique that leverages the Kronecker product structure of the Hessian matrix to reduce the memory requirements. It is generally much faster than iterative methods like conjugate gradient and Arnoldi and it allows for an easier handling of memory. Therefore, it is the only technique that can scale to very large models (e.g. billions of parameters). Its accuracy is however much worse. Let's see how it performs on our example.    </p> <pre><code>ekfac_influence_model = EkfacInfluence(\n    nn_model,\n    update_diagonal=True,\n    hessian_regularization=0.1,\n)\nekfac_influence_model = ekfac_influence_model.fit(training_data_loader)\nekfac_train_influences = ekfac_influence_model.influences(\n    *test_data, *training_data, mode=\"up\"\n)\nmean_ekfac_train_influences = np.mean(ekfac_train_influences.numpy(), axis=0)\n</code></pre> <pre>\n<code>Percentage error of EK-FAC over direct method:969.1289901733398 %\n</code>\n</pre> <p>     The accuracy is not good, and it is not recommended to use this method for small models. Nevertheless, a look at the actual influence values reveals that the EK-FAC estimates are not completely off.    </p> <p>     The above plot shows a good correlation between the EK-FAC and the direct method. Corrupted points have been circled in red, and in both the direct and approximate case they are correcly identified as having negative influence on the model's accuracy. This is confirmed by explicit calculation of the Pearson and Spearman correlation coefficients.    </p> <pre>\n<code>Pearson Correlation EK-FAC vs direct 0.9602782923532728\nSpearman Correlation EK-FAC vs direct 0.8999118321283724\n</code>\n</pre> <p>     The correlation between the EK-FAC and the direct method is quite good, and it improves significantly if we just keep top-20 highest absolute influences.    </p> <pre>\n<code>Pearson Correlation EK-FAC vs direct - top-20 influences 0.9876922383437592\nSpearman Correlation EK-FAC vs direct - top-20 influences 0.9338345864661652\n</code>\n</pre> <p>     When we calculate influence scores, typically we are more interested in assessing which training points have the highest or lowest impact on the model rather than having a precise estimate of the influence value. EK-FAC then provides a fast and memory-efficient way to calculate a coarse influence ranking of the training points which scales very well even to the largest neural networks.    </p> <p>     This was a quick introduction to the pyDVL interface for influence functions. Despite their speed and simplicity, influence functions are known to be a very noisy estimator of data quality, as pointed out in the paper           \"Influence functions in deep learning are fragile\"          . The size of the network, the weight decay, the inversion method used for calculating influences, the size of the test set: they all add up to the total amount of noise. Experiments may therefore give quantitative and qualitatively different results if not averaged across several realisations. Shapley values, on the contrary, have shown to be a more robust, but this comes at the cost of high computational requirements. PyDVL employs several parallelization and caching techniques to optimize such calculations.    </p>"},{"location":"examples/influence_wine/#influence-functions-for-outlier-detection","title":"Influence functions for outlier detection","text":"<p>     This notebook shows how to calculate influences on a NN model using pyDVL for an arbitrary dataset, and how this can be used to find anomalous or corrupted data points.    </p> <p>     It uses the wine dataset from sklearn: given a set of 13 different input parameters regarding a particular bottle, each related to some physical property (e.g. concentration of magnesium, malic acidity, alcoholic percentage, etc.), the model will need to predict to which of 3 classes the wine belongs to. For more details, please refer to the           sklearn documentation          .    </p>"},{"location":"examples/influence_wine/#imports","title":"Imports","text":""},{"location":"examples/influence_wine/#dataset","title":"Dataset","text":""},{"location":"examples/influence_wine/#fit-a-neural-network-to-the-data","title":"Fit a neural network to the data","text":"<p>     We will train a 2-layer neural network. PyDVL has some convenience wrappers to initialize a pytorch NN. If you already have a model loaded and trained, you can skip this section.    </p>"},{"location":"examples/influence_wine/#calculating-influences-for-small-neural-networks","title":"Calculating influences for small neural networks","text":"<p>     The following cell calculates the influences of each training data point on the neural network.  Neural networks have typically a very bumpy parameter space, which, during training, is explored until the configuration that minimises the loss is found. There is an important assumption in influence functions that the model lays at a (at least local) minimum of such loss, and if this is not fulfilled many issues can arise. In order to avoid this scenario, a regularisation term should be used whenever dealing with big and noisy models.    </p>"},{"location":"examples/influence_wine/#influence-of-training-features","title":"Influence of training features","text":""},{"location":"examples/influence_wine/#speeding-up-influences-for-big-models","title":"Speeding up influences for big models","text":""},{"location":"examples/influence_wine/#cg","title":"Cg","text":""},{"location":"examples/influence_wine/#lissa","title":"Lissa","text":""},{"location":"examples/influence_wine/#arnoldi","title":"Arnoldi","text":""},{"location":"examples/influence_wine/#nystrom","title":"Nystr\u00f6m","text":""},{"location":"examples/influence_wine/#ekfac","title":"EKFAC","text":""},{"location":"examples/influence_wine/#conclusions","title":"Conclusions","text":""},{"location":"examples/least_core_basic/","title":"Least Core","text":"<p>     We will be using the following functions and classes from pyDVL.    </p> <pre><code>%autoreload\nfrom pydvl.utils import (\n    Dataset,\n    Utility,\n)\nfrom pydvl.value import compute_least_core_values, LeastCoreMode, ValuationResult\nfrom pydvl.reporting.plots import shaded_mean_std\nfrom pydvl.reporting.scores import compute_removal_score\n</code></pre> <pre><code>X, y = make_classification(\n    n_samples=dataset_size,\n    n_features=50,\n    n_informative=25,\n    n_classes=3,\n    random_state=random_state,\n)\n</code></pre> <pre><code>full_dataset = Dataset.from_arrays(\n    X, y, stratify_by_target=True, random_state=random_state\n)\nsmall_dataset = Dataset.from_arrays(\n    X,\n    y,\n    stratify_by_target=True,\n    train_size=train_size,\n    random_state=random_state,\n)\n</code></pre> <pre><code>model = LogisticRegression(max_iter=500, solver=\"liblinear\")\n</code></pre> <pre><code>model.fit(full_dataset.x_train, full_dataset.y_train)\nprint(\n    f\"Training accuracy: {100 * model.score(full_dataset.x_train, full_dataset.y_train):0.2f}%\"\n)\nprint(\n    f\"Testing accuracy: {100 * model.score(full_dataset.x_test, full_dataset.y_test):0.2f}%\"\n)\n</code></pre> <pre>\n<code>Training accuracy: 86.25%\nTesting accuracy: 70.00%\n</code>\n</pre> <pre><code>model.fit(small_dataset.x_train, small_dataset.y_train)\nprint(\n    f\"Training accuracy: {100 * model.score(small_dataset.x_train, small_dataset.y_train):0.2f}%\"\n)\nprint(\n    f\"Testing accuracy: {100 * model.score(small_dataset.x_test, small_dataset.y_test):0.2f}%\"\n)\n</code></pre> <pre>\n<code>Training accuracy: 100.00%\nTesting accuracy: 47.89%\n</code>\n</pre> <pre><code>utility = Utility(model=model, data=small_dataset)\n</code></pre> <pre><code>exact_values = compute_least_core_values(\n    u=utility,\n    mode=LeastCoreMode.Exact,\n    progress=True,\n)\n</code></pre> <pre><code>exact_values_df = exact_values.to_dataframe(column=\"exact_value\").T\nexact_values_df = exact_values_df[sorted(exact_values_df.columns)]\n</code></pre> <pre><code>budget_array = np.linspace(200, 2 ** len(small_dataset), num=10, dtype=int)\n\nall_estimated_values_df = []\nall_errors = {budget: [] for budget in budget_array}\n\nfor budget in tqdm(budget_array):\n    dfs = []\n    errors = []\n    column_name = f\"estimated_value_{budget}\"\n    for i in range(20):\n        values = compute_least_core_values(\n            u=utility,\n            mode=LeastCoreMode.MonteCarlo,\n            n_iterations=budget,\n            n_jobs=n_jobs,\n        )\n        df = (\n            values.to_dataframe(column=column_name)\n            .drop(columns=[f\"{column_name}_stderr\", f\"{column_name}_updates\"])\n            .T\n        )\n        df = df[sorted(df.columns)]\n        error = mean_squared_error(\n            exact_values_df.loc[\"exact_value\"].values, np.nan_to_num(df.values.ravel())\n        )\n        all_errors[budget].append(error)\n        df[\"budget\"] = budget\n        dfs.append(df)\n    estimated_values_df = pd.concat(dfs)\n    all_estimated_values_df.append(estimated_values_df)\n\nvalues_df = pd.concat(all_estimated_values_df)\nerrors_df = pd.DataFrame(all_errors)\n</code></pre> <p>     We can see that the approximation error decreases, on average, as the we increase the budget.    </p> <p>     Still, the decrease may not always necessarily happen when we increase the number of iterations because of the fact that we sample the subsets with replacement in the Monte Carlo method i.e there may be repeated subsets.    </p> <pre><code>utility = Utility(model=model, data=full_dataset)\n</code></pre> <pre><code>method_names = [\"Random\", \"Least Core\"]\nremoval_percentages = np.arange(0, 0.41, 0.05)\n</code></pre> <pre><code>all_scores = []\n\nfor i in trange(5):\n    for method_name in method_names:\n        if method_name == \"Random\":\n            values = ValuationResult.from_random(size=len(utility.data))\n        else:\n            values = compute_least_core_values(\n                u=utility,\n                mode=LeastCoreMode.MonteCarlo,\n                n_iterations=n_iterations,\n                n_jobs=n_jobs,\n            )\n        scores = compute_removal_score(\n            u=utility,\n            values=values,\n            percentages=removal_percentages,\n            remove_best=True,\n        )\n        scores[\"method_name\"] = method_name\n        all_scores.append(scores)\n\nscores_df = pd.DataFrame(all_scores)\n</code></pre> <p>     We can clearly see that removing the most valuable data points, as given by the Least Core method, leads to, on average, a decrease in the model's performance and that the method outperforms random removal of data points.    </p> <pre><code>all_scores = []\n\nfor i in trange(5):\n    for method_name in method_names:\n        if method_name == \"Random\":\n            values = ValuationResult.from_random(size=len(utility.data))\n        else:\n            values = compute_least_core_values(\n                u=utility,\n                mode=LeastCoreMode.MonteCarlo,\n                n_iterations=n_iterations,\n                n_jobs=n_jobs,\n            )\n        scores = compute_removal_score(\n            u=utility,\n            values=values,\n            percentages=removal_percentages,\n        )\n        scores[\"method_name\"] = method_name\n        all_scores.append(scores)\n\nscores_df = pd.DataFrame(all_scores)\n</code></pre> <p>     We can clearly see that removing the least valuable data points, as given by the Least Core method, leads to, on average, an increase in the model's performance and that the method outperforms the random removal of data points.    </p>"},{"location":"examples/least_core_basic/#least-core-for-data-valuation","title":"Least Core for Data Valuation","text":"<p>     This notebook introduces Least Core methods for the computation of data values using pyDVL.    </p> <p>     Shapley values define a fair way of distributing the worth of the whole training set when every data point is part of it. But they do not consider the question of stability of subsets: Could some data points obtain a higher payoff if they formed smaller subsets? It is argued that this might be relevant if data providers are paid based on data value, since Shapley values can incentivise them not to contribute their data to the \"grand coalition\", but instead try to form smaller ones. Whether this is of actual practical relevance is debatable, but in any case, the least core is an alternative tool available for any task of Data Valuation    </p> <p>     The Core is another approach to compute data values originating in cooperative game theory that attempts to answer those questions. It is the set of feasible payoffs that cannot be improved upon by a coalition of the participants.    </p> <p>     Its use for Data Valuation was first described in the paper             If You Like Shapley Then You\u2019ll Love the Core            by Tom Yan and Ariel D. Procaccia.    </p> <p>     The Least Core value           \\(v\\)          of the           \\(i\\)          -th sample in dataset           \\(D\\)          wrt. utility           \\(u\\)          is computed by solving the following Linear Program:    </p>      \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{x_i\\in D} v_u(x_i) = u(D) &amp; \\\\ &amp; \\displaystyle\\sum_{x_i\\in S} v_u(x_i) + e \\geq u(S) &amp;, \\forall S \\subset D, S \\neq \\emptyset \\\\ \\end{array} \\]     <p>     To illustrate this method we will use a synthetic dataset. We will first use a subset of 10 data point to compute the exact values and use them to assess the Monte Carlo approximation. Afterwards, we will conduct the data removal experiments as described by Ghorbani and Zou in their paper           Data Shapley: Equitable Valuation of Data for Machine Learning          : We compute the data valuation given different computation budgets and incrementally remove a percentage of the best, respectively worst, data points and observe how that affects the utility.    </p>"},{"location":"examples/least_core_basic/#setup","title":"Setup","text":"<p>     We begin by importing the main libraries and setting some defaults.    </p>      If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/least_core_basic/#dataset","title":"Dataset","text":"<p>     We generate a synthetic dataset using the      <code>       make_classification      </code>      function from scikit-learn.    </p> <p>     We sample 200 data points from a 50-dimensional Gaussian distribution with 25 informative features and 25 non-informative features (generated as random linear combinations of the informative features).    </p> <p>     The 200 samples are uniformly distributed across 3 classes with a small percentage of noise added to the labels to make the task a bit more difficult.    </p>"},{"location":"examples/least_core_basic/#estimating-least-core-values","title":"Estimating Least Core Values","text":"<p>     In this first section we will use a smaller subset of the dataset containing 10 samples in order to be able to compute exact values in a reasonable amount of time. Afterwards, we will use the Monte Carlo method with a limited budget (maximum number of subsets) to approximate these values.    </p>"},{"location":"examples/least_core_basic/#data-removal","title":"Data Removal","text":"<p>     We now move on to the data removal experiments using the full dataset.    </p> <p>     In these experiments, we first rank the data points from most valuable  to least valuable using the values estimated by the Monte Carlo Least Core method. Then, we gradually remove from 5 to 40 percent, by increments of 5 percentage points, of the most valuable/least valuable ones, train the model on this subset and compute its accuracy.    </p>"},{"location":"examples/least_core_basic/#remove-best","title":"Remove Best","text":"<p>     We start by removing the best data points and seeing how the model's accuracy evolves.    </p>"},{"location":"examples/least_core_basic/#remove-worst","title":"Remove Worst","text":"<p>     We then proceed to removing the worst data points and seeing how the model's accuracy evolves.    </p>"},{"location":"examples/msr_banzhaf_digits/","title":"Banzhaf Semivalues","text":"<p>     We will be using the following functions from pyDVL. The main entry point is the function     <code>      compute_banzhaf_semivalues()     </code>     . In order to use it we need the classes           Dataset          ,           Utility          and           Scorer          .    </p> <pre><code>%autoreload\nfrom pydvl.reporting.plots import plot_shapley\nfrom support.banzhaf import load_digits_dataset\nfrom pydvl.value import *\n</code></pre> <pre><code>training_data, _, test_data = load_digits_dataset(\n    test_size=0.3, random_state=random_state\n)\n</code></pre> <p>     Training and test data are then used to instantiate a           Dataset          object:    </p> <pre><code>dataset = Dataset(*training_data, *test_data)\n</code></pre> <pre><code>import torch\nfrom support.banzhaf import TorchCNNModel\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = TorchCNNModel(lr=0.001, epochs=40, batch_size=32, device=device)\nmodel.fit(x=training_data[0], y=training_data[1])\n</code></pre> <pre>\n<code>Train Accuracy: 0.705\nTest Accuracy: 0.630\n</code>\n</pre> <p>     The final component is the scoring function. It can be anything like accuracy or           \\(R^2\\)          , and is set with a string from the           standard sklearn scoring methods          . Please refer to that documentation on information on how to define your own scoring function.    </p> <p>     We group dataset, model and scoring function into an instance of           Utility          and compute the Banzhaf semi-values. We take all defaults, and choose to stop computation using the           MaxChecks          stopping criterion, which terminates after a fixed number of calls to it. With the default     <code>      batch_size     </code>     of 1 this means that we will retrain the model.    </p> <p>     Note how we enable caching using memcached (assuming memcached runs with the default configuration for localhost). This is necessary in the current preliminary implementation of           permutation sampling          , which is the default for           compute_banzhaf_semivalues          .    </p> <pre><code>from pydvl.utils import MemcachedCacheBackend, MemcachedClientConfig\n\n# Compute regular Banzhaf semivalue\nutility = Utility(\n    model=model,\n    data=dataset,\n    scorer=Scorer(\"accuracy\", default=0.0, range=(0, 1)),\n    cache_backend=MemcachedCacheBackend(MemcachedClientConfig()),\n)\nvalues = compute_banzhaf_semivalues(\n    utility, done=MaxChecks(max_checks), n_jobs=n_jobs, progress=True\n)\nvalues.sort(key=\"value\")\ndf = values.to_dataframe(column=\"banzhaf_value\", use_names=True)\n</code></pre> <p>     The returned dataframe contains the mean and variance of the Monte Carlo estimates for the values:    </p>            banzhaf_value                      banzhaf_value_stderr                      banzhaf_value_updates                      156                      -1.097920                      6.662418e-02                      5                      21                      -0.925489                      1.230752e-01                      5                      152                      -0.913313                      3.358054e-02                      5                      73                      -0.778884                      3.668419e-05                      5                      85                      -0.644435                      3.454322e-08                      5           <p>     Let us plot the results. In the next cell we will take the 30 images with the lowest score and plot their values with 95% Normal confidence intervals. Keep in mind that Permutation Monte Carlo Banzhaf is typically very noisy, and it can take many steps to arrive at a clean estimate.    </p> <pre>\n<code>Average value of first 10 data points: 0.650003277874342\nExact values:\n39     0.432836\n45     0.455392\n158    0.533221\n144    0.571260\n36     0.633091\n161    0.697940\n77     0.698507\n28     0.752367\n35     0.838752\n175    0.886668\nName: banzhaf_value, dtype: float64\n</code>\n</pre> <p>     For the first 5 images, we will falsify their label, for images 6-10, we will add some noise.    </p> <pre><code>x_train_anomalous = training_data[0].copy()\ny_train_anomalous = training_data[1].copy()\nanomalous_indices = high_dvl.index.map(int).values[:10]\n\n# Set label of first 5 images to 0\ny_train_anomalous[high_dvl.index.map(int).values[:5]] = 0\n\n# Add noise to images 6-10\nindices = high_dvl.index.values[5:10].astype(int)\ncurrent_images = x_train_anomalous[indices]\nnoisy_images = current_images + 0.5 * np.random.randn(*current_images.shape)\nnoisy_images[noisy_images &amp;lt; 0] = 0.0\nnoisy_images[noisy_images &amp;gt; 1] = 1.0\nx_train_anomalous[indices] = noisy_images\n</code></pre> <pre><code>anomalous_dataset = Dataset(\n    x_train=x_train_anomalous,\n    y_train=y_train_anomalous,\n    x_test=test_data[0],\n    y_test=test_data[1],\n)\n\nanomalous_utility = Utility(\n    model=TorchCNNModel(),\n    data=anomalous_dataset,\n    scorer=Scorer(\"accuracy\", default=0.0, range=(0, 1)),\n    cache_backend=MemcachedCacheBackend(MemcachedClientConfig()),\n)\nanomalous_values = compute_banzhaf_semivalues(\n    anomalous_utility, done=MaxChecks(max_checks), n_jobs=n_jobs, progress=True\n)\nanomalous_values.sort(key=\"value\")\nanomalous_df = anomalous_values.to_dataframe(column=\"banzhaf_value\", use_names=True)\n</code></pre> <p>     Let us now take a look at the low-value images and check how many of our anomalous images are part of it.    </p> <p>     As can be seen in this figure, the valuation of the data points has decreased significantly by adding noise or falsifying their labels. This shows the potential of using Banzhaf values or other data valuation methods to detect mislabeled data points or noisy input data.    </p> <pre>\n<code>Average value of original data points: 0.650003277874342\nAverage value of modified, anomalous data points: -0.02501543656281746\nFor reference, these are the average data values of all data points used for training (anomalous):\nbanzhaf_value            0.006044\nbanzhaf_value_stderr     0.103098\nbanzhaf_value_updates    5.000000\ndtype: float64\nThese are the average data values of all points (original data):\nbanzhaf_value            0.005047\nbanzhaf_value_stderr     0.115262\nbanzhaf_value_updates    5.000000\ndtype: float64\n</code>\n</pre> <pre><code>utility = Utility(\n    model=TorchCNNModel(),\n    data=dataset,\n    scorer=Scorer(\"accuracy\", default=0.0, range=(0, 1)),\n    cache_backend=MemcachedCacheBackend(MemcachedClientConfig()),\n)\n</code></pre> <p>     Computing the values is the same, but we now use a better stopping criterion. Instead of fixing the number of utility evaluations with           MaxChecks          , we use           RankCorrelation          to stop when the change in Spearman correlation between the ranking of two successive iterations is below a threshold.    </p> <pre><code>values = compute_msr_banzhaf_semivalues(\n    utility,\n    done=RankCorrelation(rtol=0.0001, burn_in=10),\n    n_jobs=n_jobs,\n    progress=True,\n)\nvalues.sort(key=\"value\")\nmsr_df = values.to_dataframe(column=\"banzhaf_value\", use_names=True)\n</code></pre> <p>     Inspection of the values reveals (generally) much lower variances. Notice the number of updates to each value as well.    </p>            banzhaf_value                      banzhaf_value_stderr                      banzhaf_value_updates                      137                      -0.264918                      0.093597                      11                      20                      -0.217394                      0.127022                      11                      19                      -0.210309                      0.087179                      11                      41                      -0.210119                      0.071534                      11                      192                      -0.191667                      0.130774                      11           <pre><code>from sklearn.linear_model import SGDClassifier\n\nif is_CI:\n    utility = Utility(\n        model=SGDClassifier(max_iter=2),\n        data=dataset,\n        scorer=Scorer(\"accuracy\", default=0.0, range=(0, 1)),\n    )\nelse:\n    utility = Utility(\n        model=TorchCNNModel(),\n        data=dataset,\n        scorer=Scorer(\"accuracy\", default=0.0, range=(0, 1)),\n    )\n</code></pre> <pre><code>def get_semivalues_and_history(\n    sampler_t, max_checks=max_checks, n_jobs=n_jobs, progress=True\n):\n    _history = HistoryDeviation(n_steps=max_checks, rtol=1e-9)\n    if sampler_t == MSRSampler:\n        semivalue_function = compute_msr_banzhaf_semivalues\n    else:\n        semivalue_function = compute_banzhaf_semivalues\n    _values = semivalue_function(\n        utility,\n        sampler_t=sampler_t,\n        done=MaxChecks(max_checks + 2) | _history,\n        n_jobs=n_jobs,\n        progress=progress,\n    )\n    return _history, _values\n</code></pre> <pre><code># Monte Carlo Permutation Sampling Banzhaf semivalues\nhistory_permutation, permutation_values = get_semivalues_and_history(PermutationSampler)\n</code></pre> <pre><code># MSR Banzhaf values\nhistory_msr, msr_values = get_semivalues_and_history(MSRSampler)\n</code></pre> <pre><code># UniformSampler\nhistory_uniform, uniform_values = get_semivalues_and_history(UniformSampler)\n</code></pre> <pre><code># AntitheticSampler\nhistory_antithetic, antithetic_values = get_semivalues_and_history(AntitheticSampler)\n</code></pre> <pre><code># RandomHierarchicalSampler\nhistory_random, random_values = get_semivalues_and_history(RandomHierarchicalSampler)\n</code></pre> <p>     The plot above visualizes the convergence speed of different samplers used for Banzhaf semivalue calculation. /It shows the average magnitude of how much the semivalues are updated in every step of the algorithm.    </p> <p>     As you can see,             MSR            Banzhaf          stabilizes much faster. After 1000 iterations (subsets sampled and evaluated with the utility), Permutation Monte Carlo Banzhaf has evaluated the marginal function about 5 times per data point (we are using 200 data points). For           MSR          , the semivalue of each data point was updated 1000 times. Due to this, the values converge much faster wrt. the number of utility evaluations, which is the key advantage of           MSR          sampling.    </p> <p>       MSR          sampling does come at a cost, however, which is that the updates to the semivalues are more noisy than in other methods.  We will analyze the impact of this tradeoff in the next sections. First, let us look at how similar all the computed semivalues are. They are all Banzhaf values, so in a perfect world, all samplers should result in the exact same semivalues. However, due to randomness in the utility (recall that we use a neural network) and randomness in the samplers, the resulting values are likely never exactly the same. Another quality measure is that a good sampler would lead to very consistent values, a bad one to less consistent values. Let us first examine how similar the results are, then we'll look at consistency.    </p> <p>     This plot shows that the samplers lead to quite different Banzhaf semivalues, however, all of them have some points in common. The           MSR          Sampler does not seem to be significantly worse than any others.    </p> <p>     In an ideal setting without randomness, the overlap of points would be higher, however, the stochastic nature of the CNN model that we use together with the fact that we use only 200 data points for training, might overshadow these results. As a matter of fact we have the rather discouraging following result:    </p> <pre>\n<code>Total number of top 20 points that all samplers have in common: 0\n</code>\n</pre>"},{"location":"examples/msr_banzhaf_digits/#banzhaf-semi-values-for-data-valuation","title":"Banzhaf Semi-values for data valuation","text":"<p>     This notebook showcases           Data Banzhaf: A Robust Data Valuation Framework for Machine Learning          by Wang, and Jia.    </p> <p>     Computing Banzhaf semi-values using pyDVL follows basically the same procedure as all other semi-value-based methods like Shapley values. However, Data-Banzhaf tends to be more robust to stochasticity in the training process than other semi-values. A property that we study here.    </p> <p>     Additionally, we compare two sampling techniques: the standard permutation-based Monte Carlo sampling, and the so-called           MSR          (Maximum Sample Reuse) principle.    </p> <p>     In order to highlight the strengths of Data-Banzhaf, we require a stochastic model. For this reason, we use a CNN to classify handwritten digits from the           scikit-learn toy datasets          .    </p>"},{"location":"examples/msr_banzhaf_digits/#setup","title":"Setup","text":"If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/msr_banzhaf_digits/#loading-the-dataset","title":"Loading the dataset","text":"<p>     We use a support function,     <code>      load_digits_dataset()     </code>     , which downloads the data and prepares it for usage. It returns four arrays that we then use to construct a           Dataset          . The data consists of grayscale images of shape 8x8 pixels with 16 shades of gray. These images contain handwritten digits from 0 to 9.    </p>"},{"location":"examples/msr_banzhaf_digits/#creating-the-utility-and-computing-banzhaf-semivalues","title":"Creating the utility and computing Banzhaf semivalues","text":"<p>     Now we can calculate the contribution of each training sample to the model performance. First we need a model and a           Scorer          .    </p> <p>     As a model, we use a simple CNN written torch, and wrapped into an object to convert numpy arrays into tensors (as of v0.9.0 valuation methods in pyDVL work only with numpy arrays). Note that any model that implements the protocol           pydvl.utils.types.SupervisedModel          , which is just the standard sklearn interface of     <code>      fit()     </code>     ,     <code>      predict()     </code>     and     <code>      score()     </code>     can be used to construct the utility.    </p>"},{"location":"examples/msr_banzhaf_digits/#evaluation-on-anomalous-data","title":"Evaluation on anomalous data","text":"<p>     An interesting use-case for data valuation is finding anomalous data. Maybe some of the data is really noisy or has been mislabeled. To simulate this, we will change some of the labels of our dataset and add noise to some others. Intuitively, these anomalous data points should then have a lower value.    </p> <p>     To evaluate this, let us first check the average value of the first 10 data points, as these will be the ones that we modify. Currently, these are the 10 data points with the highest values:    </p>"},{"location":"examples/msr_banzhaf_digits/#maximum-sample-reuse-banzhaf","title":"Maximum Sample Reuse Banzhaf","text":"<p>     Despite the previous results already being useful, we had to retrain the model a number of times and yet the variance of the value estimates was high. This has consequences for the stability of the top-k ranking of points, which decreases the applicability of the method. We now introduce a different sampling method called Maximum Sample Reuse (           MSR          ) which reuses every sample for updating the Banzhaf values. The method was introduced by the authors of Data-Banzhaf and is much more sample-efficient, as we will show.    </p> <p>     We next construct a new utility. Note how this time we don't use a cache: the chance of hitting twice the same subset of the training set is low enough that one can dispense with it (nevertheless it can still be useful, e.g. when running many experiments).    </p>"},{"location":"examples/msr_banzhaf_digits/#compare-convergence-speed-of-banzhaf-and-msr-banzhaf-values","title":"Compare convergence speed of Banzhaf and           MSR          Banzhaf Values","text":"<p>     Conventional margin-based samplers produce require evaluating the utility twice to do one update of the value, and permutation samplers do instead           \\(n+1\\)          evaluations for           \\(n\\)          updates. Maximum Sample Reuse (           MSR          ) updates instead all indices in every sample that the utility evaluates. We compare the convergence rates of these methods.    </p> <p>     In order to do so, we will compute the semi-values using different samplers and use a high number of iterations to make sure that the values have converged.    </p>"},{"location":"examples/msr_banzhaf_digits/#similarity-of-the-semivalues-computed-using-different-samplers","title":"Similarity of the semivalues computed using different samplers","text":""},{"location":"examples/msr_banzhaf_digits/#consistency-of-the-semivalues","title":"Consistency of the semivalues","text":"<p>     Finally, we want to analyze how consistent the semivalues returned by the different samplers are. In order to do this, we compute semivalues multiple times and check how many of the data points in the top and lowest 20% of valuation of the data overlap.    </p>"},{"location":"examples/msr_banzhaf_digits/#conclusion","title":"Conclusion","text":"<p>       MSR          sampling updates the semivalue estimates for every index in the sample, much more frequently than any other sampler available, which leads to much           faster convergence          . Additionally, the sampler is more consistent with its value estimates than the other samplers, which might be caused by the higher number of value updates.    </p> <p>     There is alas no general recommendation. It is best to try different samplers when computing semivalues and test which one is best suited for your use case. Nevertheless, the           MSR          sampler seems like a more efficient sampler which may bring fast results and is well-suited for stochastic models.    </p>"},{"location":"examples/shapley_basic_spotify/","title":"Shapley values","text":"<p>     This notebook introduces Shapley methods for the computation of data value using pyDVL.    </p> <p>     In order to illustrate the practical advantages, we will predict the popularity of songs in the dataset           Top Hits Spotify from 2000-2019          , and highlight how data valuation can help investigate and boost the performance of the models. In doing so, we will describe the basic usage patterns of pyDVL.    </p> <p>     Recall that data value is a function of three things:    </p> <ol> <li>      The dataset.     </li> <li>      The model.     </li> <li>      The performance metric or scoring function.     </li> </ol> <p>     Below we will describe how to instantiate each one of these objects and how to use them for data valuation. Please also see the           documentation on data valuation          .    </p> <p>     We will be using the following functions from pyDVL. The main entry point is the function           compute_shapley_values()          , which provides a facade to all Shapley methods. In order to use it we need the classes           Dataset          ,           Utility          and           Scorer          .    </p> <pre><code>%autoreload\nfrom pydvl.reporting.plots import plot_shapley\nfrom pydvl.utils.dataset import GroupedDataset\nfrom support.shapley import load_spotify_dataset\nfrom pydvl.value import *\n</code></pre> <pre><code>training_data, val_data, test_data = load_spotify_dataset(\n    val_size=0.3, test_size=0.3, target_column=\"popularity\", random_state=random_state\n)\n</code></pre> <pre><code>training_data[0].head()\n</code></pre>            artist                      song                      duration_ms                      explicit                      year                      danceability                      energy                      key                      loudness                      mode                      speechiness                      acousticness                      instrumentalness                      liveness                      valence                      tempo                      genre                      1561                      Fetty Wap                      679 (feat. Remy Boyz)                      196693                      True                      2015                      0.618                      0.717                      7                      -5.738                      1                      0.3180                      0.00256                      0.000000                      0.6250                      0.603                      190.050                      8                      1410                      Meghan Trainor                      All About That Bass                      187920                      True                      2015                      0.807                      0.887                      9                      -3.726                      1                      0.0503                      0.05730                      0.000003                      0.1240                      0.961                      134.052                      14                      1772                      Katy Perry                      Chained To The Rhythm                      237733                      False                      2017                      0.562                      0.800                      0                      -5.404                      1                      0.1120                      0.08140                      0.000000                      0.1990                      0.471                      95.029                      14                      1670                      Sigala                      Sweet Lovin' - Radio Edit                      202149                      False                      2015                      0.683                      0.910                      10                      -1.231                      1                      0.0515                      0.05530                      0.000005                      0.3360                      0.674                      124.977                      15                      1780                      Liam Payne                      Strip That Down                      204502                      False                      2017                      0.869                      0.485                      6                      -5.595                      1                      0.0545                      0.24600                      0.000000                      0.0765                      0.527                      106.028                      14           <p>     The dataset has many high-level features, some quite intuitive ('duration_ms' or 'tempo'), while others are a bit more cryptic ('valence'?). For information on each feature, please consult           the dataset's website          .    </p> <p>     In our analysis, we will use all the columns, except for 'artist' and 'song', to predict the 'popularity' of each song. We will nonetheless keep the information on song and artist in a separate object for future reference.    </p> <pre><code>song_name = training_data[0][\"song\"]\nartist = training_data[0][\"artist\"]\ntraining_data[0] = training_data[0].drop([\"song\", \"artist\"], axis=1)\ntest_data[0] = test_data[0].drop([\"song\", \"artist\"], axis=1)\nval_data[0] = val_data[0].drop([\"song\", \"artist\"], axis=1)\n</code></pre> <p>     Input and label data are then used to instantiate a           Dataset          object:    </p> <pre><code>dataset = Dataset(*training_data, *val_data)\n</code></pre> <p>     The calculation of exact Shapley values is computationally very expensive (exponentially so!) because it requires training the model on every possible subset of the training set. For this reason, PyDVL implements techniques to speed up the calculation, such as           Monte Carlo approximations          ,           surrogate models          or           caching          of intermediate results and grouping of data to calculate group Shapley values instead of single data points.    </p> <p>     In our case, we will group songs by artist and calculate the Shapley value for the artists. Given the           pandas Series          for 'artist', to group the dataset by it, one does the following:    </p> <pre><code>grouped_dataset = GroupedDataset.from_dataset(dataset=dataset, data_groups=artist)\n</code></pre> <pre><code>utility = Utility(\n    model=GradientBoostingRegressor(n_estimators=3),\n    data=grouped_dataset,\n    scorer=Scorer(\"neg_mean_absolute_error\", default=0.0),\n)\nvalues = compute_shapley_values(\n    utility,\n    mode=ShapleyMode.TruncatedMontecarlo,\n    # Stop if the standard error is below 1% of the range of the values (which is ~2),\n    # or if the number of updates exceeds 1000\n    done=AbsoluteStandardError(threshold=0.2, fraction=0.9) | MaxUpdates(1000),\n    truncation=RelativeTruncation(utility, rtol=0.01),\n    n_jobs=-1,\n)\nvalues.sort(key=\"value\")\ndf = values.to_dataframe(column=\"data_value\", use_names=True)\n</code></pre> <pre>\n<code>Cancellation of futures is not supported by the joblib backend\n</code>\n</pre> <p>     The function           compute_shapley_values()          serves as a common access point to all Shapley methods. For most of them, we must choose a     <code>      StoppingCriterion     </code>     with the argument     <code>      done=     </code>     . In this case we choose to stop when the ratio of standard error to value is below 0.2 for at least 90% of the training points, or if the number of updates of any index exceeds 1000. The     <code>      mode     </code>     argument specifies the Shapley method to use. In this case, we use the           Truncated Monte Carlo approximation          , which is the fastest of the Monte Carlo methods, owing both to using the permutation definition of Shapley values and the ability to truncate the iteration over a given permutation. We configure this to happen when the contribution of the remaining elements is below 1% of the total utility with the parameter     <code>      truncation=     </code>     and the policy           RelativeTruncation          .    </p> <p>     Let's take a look at the returned dataframe:    </p> <pre><code>df.head()\n</code></pre>            data_value                      data_value_stderr                      Years &amp; Years                      -1.150663                      0.195376                      Reik                      -1.123071                      0.126558                      Astrid S                      -0.945702                      0.331619                      Liam Payne                      -0.886687                      0.112654                      DB Boulevard                      -0.847957                      0.057503           <p>     The first thing to notice is that we sorted the results in ascending order of Shapley value. The index holds the labels for each data group: in this case, artist names. The column     <code>      data_value     </code>     is just that: the Shapley Data value, and     <code>      data_value_stderr     </code>     is its estimated standard error because we are using a Monte Carlo approximation.    </p> <p>     Let us plot the results. In the next cell we will take the 30 artists with the lowest score and plot their values with 95% Normal confidence intervals. Keep in mind that Monte Carlo Shapley is typically very noisy, and it can take many steps to arrive at a clean estimate.    </p> <p>     We can immediately see that many artists (groups of samples) have very low, even negative value, which means that they tend to decrease the total score of the model when present in the training set! What happens if we remove them?    </p> <p>     In the next cell we create a new training set excluding the artists with the lowest scores:    </p> <pre><code>low_dvl_artists = df.iloc[: int(0.2 * len(df))].index.to_list()\nartist_filter = ~artist.isin(low_dvl_artists)\nX_train_good_dvl = training_data[0][artist_filter]\ny_train_good_dvl = training_data[1][artist_filter]\n</code></pre> <p>     Now we will use this \"cleaned\" dataset to retrain the same model and compare its mean absolute error to the one trained on the full dataset. Notice that the score now is calculated using the test set, while in the calculation of the Shapley values we were using the validation set.    </p> <pre><code>model_good_data = GradientBoostingRegressor(n_estimators=3).fit(\n    X_train_good_dvl, y_train_good_dvl\n)\nerror_good_data = mean_absolute_error(\n    model_good_data.predict(test_data[0]), test_data[1]\n)\n\nmodel_all_data = GradientBoostingRegressor(n_estimators=3).fit(\n    training_data[0], training_data[1]\n)\nerror_all_data = mean_absolute_error(model_all_data.predict(test_data[0]), test_data[1])\n\nprint(f\"Improvement: {100*(error_all_data - error_good_data)/error_all_data:02f}%\")\n</code></pre> <pre>\n<code>Improvement: 15.314214%\n</code>\n</pre> <p>     The score has improved by almost 14%! This is quite an important result, as it shows a consistent process to improve the performance of a model by excluding data points from its training set.    </p>      One must however proceed with caution instead of simply throwing away data. For one, `mean_absolute_error` is an estimate of generalization error on unseen data, so the improvement we see on the test set might not be as large upon deployment. It would be advisable to cross-validate this whole process to obtain more conservative estimates. It is also advisable to manually inspect the artists with low value and to try to understand the reason why the model behaves like it does. Finally, remember that **the value depends on the model chosen**! Artists that are detrimental to the Gradient Boosting Regressor might be informative for a different model (although it is likely that the worst ones share some characteristic making them \"bad\" for other regressors).     <p>     Let us take all the songs by Billie Eilish, set their score to 0 and re-calculate the Shapley values.    </p> <pre><code>y_train_anomalous = training_data[1].copy(deep=True)\ny_train_anomalous[artist == \"Billie Eilish\"] = 0\nanomalous_dataset = Dataset(\n    x_train=training_data[0],\n    y_train=y_train_anomalous,\n    x_test=val_data[0],\n    y_test=val_data[1],\n)\ngrouped_anomalous_dataset = GroupedDataset.from_dataset(anomalous_dataset, artist)\nanomalous_utility = Utility(\n    model=GradientBoostingRegressor(n_estimators=3),\n    data=grouped_anomalous_dataset,\n    scorer=Scorer(\"neg_mean_absolute_error\", default=0.0),\n)\nvalues = compute_shapley_values(\n    anomalous_utility,\n    mode=ShapleyMode.TruncatedMontecarlo,\n    done=AbsoluteStandardError(threshold=0.2, fraction=0.9) | MaxUpdates(1000),\n    n_jobs=-1,\n)\nvalues.sort(key=\"value\")\ndf = values.to_dataframe(column=\"data_value\", use_names=True)\n</code></pre> <pre>\n<code>Cancellation of futures is not supported by the joblib backend\n</code>\n</pre> <p>     Let us now consider the low-value artists (at least for predictive purposes, no claims are made about their artistic value!) and plot the results    </p> <p>     And Billie Eilish (our anomalous data group) has moved from top contributor to having negative impact on the performance of the model, as expected!    </p> <p>     What is going on? A popularity of 0 for Billie Eilish's songs is inconsistent with listening patterns for other artists. In artificially setting this, we degrade the predictive power of the model.    </p> <p>     By dropping low-value groups or samples, one can often increase model performance, but by           inspecting          them, it is possible to identify bogus data sources or acquisition methods.    </p>"},{"location":"examples/shapley_basic_spotify/#shapley-for-data-valuation","title":"Shapley for data valuation","text":""},{"location":"examples/shapley_basic_spotify/#setup","title":"Setup","text":"<p>     We begin by importing the main libraries and setting some defaults.    </p>      If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_basic_spotify/#loading-and-grouping-the-dataset","title":"Loading and grouping the dataset","text":"<p>     pyDVL provides a support function for this notebook,     <code>      load_spotify_dataset()     </code>     , which downloads data on songs published after 2014, and splits 30% of data for testing, and 30% of the remaining data for validation. The return value is a triple of training, validation and test data as lists of the form     <code>      [X_input, Y_label]     </code>     .    </p>"},{"location":"examples/shapley_basic_spotify/#creating-the-utility-and-computing-values","title":"Creating the utility and computing values","text":"<p>     Now we can calculate the contribution of each group to the model performance.    </p> <p>     As a model, we use scikit-learn's           GradientBoostingRegressor          , but pyDVL can work with any model from sklearn, xgboost or lightgbm. More precisely, any model that implements the protocol           pydvl.utils.types.SupervisedModel          , which is just the standard sklearn interface of     <code>      fit()     </code>     ,     <code>      predict()     </code>     and     <code>      score()     </code>     can be used to construct the utility.    </p> <p>     The third and final component is the scoring function. It can be anything like accuracy or           \\(R^2\\)          , and is set with a string from the           standard sklearn scoring methods          . Please refer to that documentation on information on how to define your own scoring function.    </p> <p>     We group dataset, model and scoring function into an instance of           Utility          .    </p>"},{"location":"examples/shapley_basic_spotify/#evaluation-on-anomalous-data","title":"Evaluation on anomalous data","text":"<p>     One interesting test is to corrupt some data and to monitor how their value changes. To do this, we will take one of the artists with the highest value and set the popularity of all their songs to 0.    </p>"},{"location":"examples/shapley_knn_flowers/","title":"KNN Shapley","text":"<p>     This notebook shows how to calculate Shapley values for the K-Nearest Neighbours algorithm. By making use of the local structure of KNN, it is possible to compute an exact value in almost linear time, as opposed to exponential complexity of exact, model-agnostic Shapley.    </p> <p>     The main idea is to exploit the fact that adding or removing points beyond the k-ball doesn't influence the score. Because the algorithm then essentially only needs to do a search it runs in           \\(\\mathcal{O}(N \\log N)\\)          time.    </p> <p>     By further using approximate nearest neighbours, it is possible to achieve           \\((\\epsilon,\\delta)\\)          -approximations in sublinear time. However, this is not implemented in pyDVL yet.    </p> <p>     We refer to the original paper that pyDVL implements for details:           Jia, Ruoxi, David Dao, Boxin Wang, Frances Ann Hubis, Nezihe Merve Gurel, Bo Li, Ce Zhang, Costas Spanos, and Dawn Song.             Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms            . Proceedings of the VLDB Endowment 12, no. 11 (1 July 2019): 1610\u201323.      </p> <p>     The main entry point is the function           compute_shapley_values()          , which provides a facade to all Shapley methods. In order to use it we need the classes           Dataset          ,           Utility          and           Scorer          , all of which can be imported from     <code>      pydvl.value     </code>     :    </p> <pre><code>from pydvl.value import *\n</code></pre> <pre><code>sklearn_dataset = datasets.load_iris()\ndata = Dataset.from_sklearn(sklearn_dataset)\nknn = sk.neighbors.KNeighborsClassifier(n_neighbors=5)\nutility = Utility(knn, data)\n</code></pre> <pre><code>shapley_values = compute_shapley_values(utility, mode=ShapleyMode.KNN, progress=True)\nshapley_values.sort(key=\"value\")\nvalues = shapley_values.values\n</code></pre> <pre>\n<code>0it [00:00, ?it/s]</code>\n</pre> <p>     If we now look at  the distribution of Shapley values for each class, we see that each has samples with both high and low scores. This is expected, because an accurate model uses information of all classes.    </p> <pre><code>corrupted_data = deepcopy(data)\nn_corrupted = 10\ncorrupted_data.y_train[:n_corrupted] = (corrupted_data.y_train[:n_corrupted] + 1) % 3\nknn = sk.neighbors.KNeighborsClassifier(n_neighbors=5)\ncontaminated_values = compute_shapley_values(\n    Utility(knn, corrupted_data), mode=ShapleyMode.KNN\n)\n</code></pre> <p>     Taking the average corrupted value and comparing it to non-corrupted ones, we notice that on average anomalous points have a much lower score, i.e. they tend to be much less valuable to the model.    </p> <p>     To do this, first we make sure that we access the results by data index with a call to     <code>      ValuationResult.sort()     </code>     , then we split the values into two groups: corrupted and non-corrupted. Note how we access property     <code>      values     </code>     of the     <code>      ValuationResult     </code>     object. This is a numpy array of values, sorted however the object was sorted. Finally, we compute the quantiles of the two groups and compare them. We see that the corrupted mean is in the lowest percentile of the value distribution, while the correct mean is in the 70th percentile.    </p> <pre><code>contaminated_values.sort(\n    key=\"index\"\n)  # This is redundant, but illustrates sorting, which is in-place\n\ncorrupted_shapley_values = contaminated_values.values[:n_corrupted]\ncorrect_shapley_values = contaminated_values.values[n_corrupted:]\n\nmean_corrupted = np.mean(corrupted_shapley_values)\nmean_correct = np.mean(correct_shapley_values)\npercentile_corrupted = np.round(100 * np.mean(values &amp;lt; mean_corrupted), 0)\npercentile_correct = np.round(100 * np.mean(values &amp;lt; mean_correct), 0)\n\nprint(\n    f\"The corrupted mean is at percentile {percentile_corrupted:.0f} of the value distribution.\"\n)\nprint(\n    f\"The correct mean is percentile {percentile_correct:.0f} of the value distribution.\"\n)\n</code></pre> <pre>\n<code>The corrupted mean is at percentile 2 of the value distribution.\nThe correct mean is percentile 71 of the value distribution.\n</code>\n</pre> <p>     This is confirmed if we plot the distribution of Shapley values and circle corrupt points in red. They all tend to have low Shapley scores, regardless of their position in space and assigned label:    </p>"},{"location":"examples/shapley_knn_flowers/#knn-shapley","title":"KNN Shapley","text":""},{"location":"examples/shapley_knn_flowers/#setup","title":"Setup","text":"<p>     We begin by importing the main libraries and setting some defaults.    </p>      If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_knn_flowers/#building-a-dataset-and-a-utility","title":"Building a Dataset and a Utility","text":"<p>     We use           the sklearn iris dataset          and wrap it into a           pydvl.utils.dataset.Dataset          calling the factory           pydvl.utils.dataset.Dataset.from_sklearn()          . This automatically creates a train/test split for us which will be used to compute the utility.    </p> <p>     We then create a model and instantiate a           Utility          using data and model. The model needs to implement the protocol           pydvl.utils.types.SupervisedModel          , which is just the standard sklearn interface of     <code>      fit()     </code>     ,     <code>      predict()     </code>     and     <code>      score()     </code>     . In constructing the     <code>      Utility     </code>     one can also choose a scoring function, but we pick the default which is just the model's     <code>      knn.score()     </code>     .    </p>"},{"location":"examples/shapley_knn_flowers/#computing-values","title":"Computing values","text":"<p>     Calculating the Shapley values is straightforward. We just call           compute_shapley_values()          with the utility object we created above. The function returns a           ValuationResult          . This object contains the values themselves, data indices and labels.    </p>"},{"location":"examples/shapley_knn_flowers/#inspecting-the-results","title":"Inspecting the results","text":"<p>     Let us first look at the labels' distribution as a function of petal and sepal length:    </p>"},{"location":"examples/shapley_knn_flowers/#corrupting-labels","title":"Corrupting labels","text":"<p>     To test how informative values are, we can corrupt some training labels and see how their Shapley values change with respect to the non-corrupted points.    </p>"},{"location":"examples/shapley_utility_learning/","title":"Data utility learning","text":"<p>     This notebook introduces           Data Utility Learning          , a method of approximating Data Shapley values by learning to estimate the utility function.    </p> <p>     The idea is to employ a model to learn the performance of the learning algorithm of interest on unseen data combinations (i.e. subsets of the dataset). The method was originally described in           Wang, Tianhao, Yu Yang, and Ruoxi Jia.             Improving Cooperative Game Theory-Based Data Valuation via Data Utility Learning            . arXiv, 2022          .    </p> <p>        Warning:            Work on Data Utility Learning is preliminary. It remains to be seen when or whether it can be put effectively into application. For this further testing and benchmarking are required.     </p> <p>     Recall the definition of Shapley value           \\(v_u(i)\\)          for data point           \\(i\\)          :    </p>      \\[\\begin{equation} v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{|S|}^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)] , \\tag{1} \\label{eq:shapley-def} \\end{equation}\\]     <p>     where           \\(N\\)          is the set of all indices in the training set and           \\(u\\)          is the utility.    </p> <p>     In Data Utility Learning, to avoid the exponential cost of computing this sum, one learns a surrogate model for           \\(u\\)          . We start by sampling so-called           utility samples          to form a training set           \\(S_\\mathrm{train}\\)          for our utility model. Each utility sample is a tuple consisting of a subset of indices           \\(S_j\\)          in the dataset and its utility           \\(u(S_j)\\)          :    </p>      \\[\\mathcal{S}_\\mathrm{train} = \\{(S_j, u(S_j): j = 1 , ..., m_\\mathrm{train}\\}\\]     <p>     where           \\(m_\\mathrm{train}\\)          denotes the           training budget          for the learned utility function.    </p> <p>     The subsets are then transformed into boolean vectors           \\(\\phi\\)          in which a           \\(1\\)          at index           \\(k\\)          means that the           \\(k\\)          -th sample of the dataset is present in the subset:    </p>      \\[S_j \\mapsto \\phi_j \\in \\{ 0, 1 \\}^{N}\\]     <p>     We fit a regression model           \\(\\tilde{u}\\)          , called           data utility model          , on the transformed utility samples           \\(\\phi (\\mathcal{S}_\\mathrm{train}) := \\{(\\phi(S_j), u(S_j): j = 1 , ..., m_\\mathrm{train}\\}\\)          and use it to predict instead of computing the utility for any           \\(S_j \\notin \\mathcal{S}_\\mathrm{train}\\)          . We abuse notation and identify           \\(\\tilde{u}\\)          with the composition           \\(\\tilde{u} \\circ \\phi : N \\rightarrow \\mathbb{R}\\)          .    </p> <p>     The main assumption is that it is much faster to fit and use           \\(\\tilde{u}\\)          than it is to compute           \\(u\\)          and that for most           \\(i\\)          ,           \\(v_\\tilde{u}(i) \\approx v_u(i)\\)          in some sense.    </p> <p>     As is the case with all other Shapley methods, the main entry point is the function           compute_shapley_values()          , which provides a facade to all algorithms in this family. We use it with the usual classes           Dataset          and           Utility          . In addition, we must import the core class for learning a utility,           DataUtilityLearning          .    </p> <pre><code>%autoreload\nfrom pydvl.utils import DataUtilityLearning, top_k_value_accuracy\nfrom pydvl.reporting.plots import shaded_mean_std\nfrom pydvl.value import *\n</code></pre> <pre><code>dataset = Dataset.from_sklearn(\n    load_iris(),\n    train_size=train_size,\n    random_state=random_state,\n    stratify_by_target=True,\n)\n</code></pre> <p>     We verify that, as in the paper, if we fit a Support-Vector Classifier to the training data, we obtain an accuracy of around 92%:    </p> <pre><code>model = LinearSVC()\nmodel.fit(dataset.x_train, dataset.y_train)\nprint(f\"Mean accuracy: {100 * model.score(dataset.x_test, dataset.y_test):0.2f}%\")\n</code></pre> <pre>\n<code>Mean accuracy: 92.59%\n</code>\n</pre> <pre><code>computation_times = {}\n</code></pre> <pre><code>utility = Utility(model=model, data=dataset)\n</code></pre> <pre><code>start_time = time.monotonic()\n\nresult = compute_shapley_values(\n    u=utility,\n    mode=ShapleyMode.CombinatorialExact,\n    n_jobs=-1,\n    progress=False,\n)\n\ncomputation_time = time.monotonic() - start_time\ncomputation_times[\"exact\"] = computation_time\n\ndf = result.to_dataframe(column=\"exact\").drop(columns=[\"exact_stderr\"])\n</code></pre> <p>     We now estimate the Data Shapley values using the           DataUtilityLearning          wrapper. This class wraps a           Utility          and delegates calls to it, up until a given budget. Every call yields a utility sample which is saved under the hood for training of the given utility model. Once the budget is exhausted,     <code>      DataUtilityLearning     </code>     fits the model to the utility samples and all subsequent calls use the learned model to predict the wrapped utility instead of delegating to it.    </p> <p>     For the utility model we follow the paper and use a fully connected neural network. To train it we use a total of     <code>      training_budget     </code>     utility samples. We repeat this multiple times for each training budget.    </p>      Note how we use a MonteCarlo approximation instead of `combinatorial_exact` as before. This is because the exact computation samples subsets in a particular order, from the lowest size to the largest. Because the training budget for the model to learn the utility is around 1/4th of the total number of subsets, this would mean that we would never see utility samples for the larger sizes and the model would be biased (try it!)     <pre><code>mlp_kwargs = dict(\n    hidden_layer_sizes=(20, 10),\n    activation=\"relu\",\n    solver=\"adam\",\n    learning_rate_init=0.001,\n    batch_size=batch_size,\n    max_iter=800,\n)\n\nprint(\n    f\"Doing {n_runs} runs for each of {len(training_budget_values)} different training budgets.\"\n)\n\npbar = tqdm(\n    product(range(n_runs), training_budget_values),\n    total=n_runs * len(training_budget_values),\n)\nfor idx, budget in pbar:\n    pbar.set_postfix_str(f\"Run {idx} for training budget: {budget}\")\n    dul_utility = DataUtilityLearning(\n        u=utility, training_budget=budget, model=MLPRegressor(**mlp_kwargs)\n    )\n\n    start_time = time.monotonic()\n\n    # DUL will kick in after training_budget calls to utility\n    result = compute_shapley_values(\n        u=dul_utility,\n        mode=ShapleyMode.PermutationMontecarlo,\n        done=MaxUpdates(300),\n        n_jobs=-1,\n    )\n\n    computation_time = time.monotonic() - start_time\n    if budget in computation_times:\n        computation_times[budget].append(computation_time)\n    else:\n        computation_times[budget] = [computation_time]\n\n    dul_df = result.to_dataframe(column=f\"{budget}_{idx}\").drop(\n        columns=[f\"{budget}_{idx}_stderr\"]\n    )\n    df = pd.concat([df, dul_df], axis=1)\n\ncomputation_times_df = pd.DataFrame(computation_times)\n</code></pre> <pre>\n<code>Doing 10 runs for each of 10 different training budgets.\n</code>\n</pre> <pre>\n<code>  0%|          | 0/100 [00:00&lt;?, ?it/s]</code>\n</pre> <p>     Next we compute the           \\(l_1\\)          error for the different training budgets across all runs and plot mean and standard deviation. We obtain results analogous to Figure 1 of the paper, verifying that the method indeed works for estimating the Data Shapley values (at least in this context).    </p> <p>     In the plot we also display the mean and standard deviation of the computation time taken for each training budget.    </p> <pre><code>errors = np.zeros((len(training_budget_values), n_runs), dtype=float)\naccuracies = np.zeros((len(training_budget_values), n_runs), dtype=float)\n\ntop_k = 3\n\nfor i, budget in enumerate(training_budget_values):\n    for j in range(n_runs):\n        y_true = df[\"exact\"].values\n        y_estimated = df[f\"{budget}_{j}\"].values\n        errors[i, j] = np.linalg.norm(y_true - y_estimated, ord=2)\n        accuracies[i, j] = top_k_value_accuracy(y_true, y_estimated, k=top_k)\n\nerror_from_mean = np.linalg.norm(df[\"exact\"].values - df[\"exact\"].values.mean(), ord=2)\n</code></pre> <p>     Let us next look at how well the ranking of values resulting from using the surrogate           \\(\\tilde{u}\\)          matches the ranking by the exact values. For this we fix           \\(k=3\\)          and consider the           \\(k\\)          samples with the highest value according to           \\(\\tilde{u}\\)          and           \\(u\\)          :    </p> <p>     Finally, for each sample, we look at the distance of the estimates to the exact value across runs. Boxes are centered at the 50th percentile with wiskers at the 25th and 75th. We plot relative distances, as a percentage. We observe a general tendency to underestimate the value:    </p> <pre><code>highest_value_index = df.index[df[\"exact\"].argmax()]\ny_train_corrupted = dataset.y_train.copy()\ny_train_corrupted[highest_value_index] = (\n    y_train_corrupted[highest_value_index] + 1\n) % 3\n\ncorrupted_dataset = Dataset(\n    x_train=dataset.x_train,\n    y_train=y_train_corrupted,\n    x_test=dataset.x_test,\n    y_test=dataset.y_test,\n)\n</code></pre> <p>     We retrain the model on the new dataset and verify that the accuracy decreases:    </p> <pre><code>model = LinearSVC()\nmodel.fit(dataset.x_train, y_train_corrupted)\nprint(f\"Mean accuracy: {100 * model.score(dataset.x_test, dataset.y_test):0.2f}%\")\n</code></pre> <pre>\n<code>Mean accuracy: 82.96%\n</code>\n</pre> <p>     Finally, we recompute the values of all samples using the exact method and the best training budget previously obtained and then plot the resulting scores.    </p> <pre><code>best_training_budget = training_budget_values[errors.mean(axis=1).argmin()]\n\nutility = Utility(\n    model=LinearSVC(),\n    data=corrupted_dataset,\n)\n\nresult = compute_shapley_values(\n    u=utility,\n    mode=ShapleyMode.CombinatorialExact,\n    n_jobs=-1,\n    progress=False,\n)\ndf_corrupted = result.to_dataframe(column=\"exact\").drop(columns=[\"exact_stderr\"])\n\ndul_utility = DataUtilityLearning(\n    u=utility, training_budget=best_training_budget, model=MLPRegressor(**mlp_kwargs)\n)\n\nresult = compute_shapley_values(\n    u=dul_utility,\n    mode=ShapleyMode.PermutationMontecarlo,\n    done=MaxUpdates(300),\n    n_jobs=-1,\n)\ndul_df = result.to_dataframe(column=\"estimated\").drop(columns=[\"estimated_stderr\"])\ndf_corrupted = pd.concat([df_corrupted, dul_df], axis=1)\n</code></pre> <p>     We can see in the figure that both methods assign the lowest value to the sample with the corrupted label.    </p>      As mentioned above, despite the previous results, this work is preliminary and the usefulness of Data Utility Learning remains to be tested in practice."},{"location":"examples/shapley_utility_learning/#data-utility-learning","title":"Data Utility Learning","text":""},{"location":"examples/shapley_utility_learning/#setup","title":"Setup","text":"<p>     We begin by importing the main libraries and setting some defaults.    </p>      If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_utility_learning/#dataset","title":"Dataset","text":"<p>     Following the paper, we take 15 samples (10%) from the           Iris dataset          and compute their Data Shapley values by using all the remaining samples as test set for computing the utility, which in this case is accuracy.    </p>"},{"location":"examples/shapley_utility_learning/#data-shapley","title":"Data Shapley","text":"<p>     We start by defining the utility using the model and computing the exact Data Shapley values by definition           \\(\\ref{eq:shapley-def}\\)          .    </p>"},{"location":"examples/shapley_utility_learning/#evaluation-on-anomalous-data","title":"Evaluation on anomalous data","text":"<p>     One interesting way to assess the Data Utility Learning approach is to corrupt some data and monitor how the value changes. To do this, we will take the sample with the highest score and change its label.    </p>"},{"location":"getting-started/","title":"Getting started","text":"<p>If you want to jump straight in, install pyDVL and then check out the examples. You will probably want to install with support for influence function computation.</p> <p>We have introductions to the ideas behind Data valuation and Influence functions, as well as a short overview of common applications.</p>"},{"location":"getting-started/#","title":"Installing pyDVL","text":"<p>To install the latest release use:</p> <pre><code>pip install pyDVL\n</code></pre> <p>See Extras for optional dependencies, in particular if you are interested in influence functions. You can also install the latest development version from TestPyPI:</p> <pre><code>pip install pyDVL --index-url https://test.pypi.org/simple/\n</code></pre> <p>In order to check the installation you can use:</p> <pre><code>python -c \"import pydvl; print(pydvl.__version__)\"\n</code></pre>"},{"location":"getting-started/#dependencies","title":"Dependencies","text":"<p>pyDVL requires Python &gt;= 3.8, numpy, scikit-learn, scipy, cvxpy for the core methods, and joblib for parallelization locally. Additionally,the Influence functions module requires PyTorch (see Extras below).</p>"},{"location":"getting-started/#installation-extras","title":"Extras","text":"<p>pyDVL has a few extra dependencies that can be optionally installed:</p>"},{"location":"getting-started/#installation-influences","title":"Influence functions","text":"<p>To use the module on influence functions, pydvl.influence, run:</p> <pre><code>pip install pyDVL[influence]\n</code></pre> <p>This includes a dependency on PyTorch (Version 2.0 and above) and thus is left out by default.</p>"},{"location":"getting-started/#cupy","title":"CuPy","text":"<p>In case that you have a supported version of CUDA installed (v11.2 to 11.8 as of this writing), you can enable eigenvalue computations for low-rank approximations with CuPy on the GPU by using:</p> <pre><code>pip install pyDVL[cupy]\n</code></pre> <p>This installs cupy-cuda11x.</p> <p>If you use a different version of CUDA, please install CuPy manually.</p>"},{"location":"getting-started/#ray","title":"Ray","text":"<p>If you want to use Ray to distribute data valuation workloads across nodes in a cluster (it can be used locally as well, but for this we recommend joblib instead) install pyDVL using:</p> <pre><code>pip install pyDVL[ray]\n</code></pre> <p>see the intro to parallelization for more details on how to use it.</p>"},{"location":"getting-started/#memcached","title":"Memcached","text":"<p>If you want to use Memcached for caching utility evaluations, use:</p> <pre><code>pip install pyDVL[memcached]\n</code></pre> <p>This installs pymemcache additionally. Be aware that you still have to start a memcached server manually. See Setting up the Memcached cache.</p>"},{"location":"getting-started/advanced-usage/","title":"Advanced usage","text":"<p>Besides the dos and don'ts of data valuation itself, which are the subject of the examples and the documentation of each method, there are two main things to keep in mind when using pyDVL namely Parallelization and Caching.</p>"},{"location":"getting-started/advanced-usage/#setting-up-parallelization","title":"Parallelization","text":"<p>pyDVL uses parallelization to scale and speed up computations. It does so using one of Dask, Ray or Joblib. The first is used in the influence package whereas the other two are used in the value package.</p>"},{"location":"getting-started/advanced-usage/#data-valuation","title":"Data valuation","text":"<p>For data valuation, pyDVL uses joblib for local parallelization (within one machine) and supports using Ray for distributed parallelization (across multiple machines).</p> <p>The former works out of the box but for the latter you will need to install additional dependencies (see Extras) and to provide a running cluster (or run ray in local mode).</p> <p>Info</p> <p>As of v0.9.0 pyDVL does not allow requesting resources per task sent to the cluster, so you will need to make sure that each worker has enough resources to handle the tasks it receives. A data valuation task using game-theoretic methods will typically make a copy of the whole model and dataset to each worker, even if the re-training only happens on a subset of the data. This means that you should make sure that each worker has enough memory to handle the whole dataset.</p> <p>We use backend classes for both joblib and ray as well as two types of executors for the different algorithms: the first uses a map reduce pattern as seen in the MapReduceJob class and the second implements the futures executor interface from concurrent.futures.</p> <p>As a convenience, you can also instantiate a parallel backend class by using the init_parallel_backend function:</p> <pre><code>from pydvl.parallel import init_parallel_backend\nparallel_backend = init_parallel_backend(backend_name=\"joblib\")\n</code></pre> <p>Info</p> <p>The executor classes are not meant to be instantiated and used by users of pyDVL. They are used internally as part of the computations of the  different methods.</p> <p>Deprecation notice</p> <p>We are currently planning to deprecate MapReduceJob in favour of the futures executor interface because it allows for more diverse computation patterns with interruptions.</p>"},{"location":"getting-started/advanced-usage/#joblib","title":"Joblib","text":"<p>Please follow the instructions in Joblib's documentation for all possible configuration options that you can pass to the parallel_config context manager.</p> <p>To use the joblib parallel backend with the <code>loky</code> backend and verbosity set to <code>100</code> to compute exact shapley values you would use:</p> <pre><code>import joblib\nfrom pydvl.parallel import JoblibParallelBackend\nfrom pydvl.value.shapley import combinatorial_exact_shapley\nfrom pydvl.utils.utility import Utility\n\nparallel_backend = JoblibParallelBackend() \nu = Utility(...)\n\nwith joblib.parallel_config(backend=\"loky\", verbose=100):\n    values = combinatorial_exact_shapley(u, parallel_backend=parallel_backend)\n</code></pre>"},{"location":"getting-started/advanced-usage/#ray","title":"Ray","text":"<p>Additional dependencies</p> <p>The Ray parallel backend requires optional dependencies. See Extras for more information.</p> <p>Please follow the instructions in Ray's documentation to set up a remote cluster. You could alternatively use a local cluster and in that case you don't have to set anything up.</p> <p>Before starting a computation, you should initialize ray by calling  <code>ray.init</code> with the appropriate parameters:</p> <p>To set up and start a local ray cluster with 4 CPUs you would use:</p> <pre><code>import ray\n\nray.init(num_cpus=4)\n</code></pre> <p>Whereas for a remote ray cluster you would use:</p> <pre><code>import ray\n\naddress = \"&lt;Hypothetical Ray Cluster IP Address&gt;\"\nray.init(address)\n</code></pre> <p>To use the ray parallel backend to compute exact shapley values you would use:</p> <pre><code>import ray\nfrom pydvl.parallel import RayParallelBackend\nfrom pydvl.value.shapley import combinatorial_exact_shapley\nfrom pydvl.utils.utility import Utility\n\nray.init()\nparallel_backend = RayParallelBackend()\nu = Utility(...)\nvaues = combinatorial_exact_shapley(u, parallel_backend=parallel_backend)\n</code></pre>"},{"location":"getting-started/advanced-usage/#futures-executor","title":"Futures executor","text":"<p>For the futures executor interface, we have implemented an executor  class for ray in RayExecutor and rely on joblib's loky get_reusable_executor function to instantiate an executor for local parallelization.</p> <p>They are both compatibles with the builtin ThreadPoolExecutor and ProcessPoolExecutor classes.</p> <pre><code>&gt;&gt;&gt; from joblib.externals.loky import _ReusablePoolExecutor\n&gt;&gt;&gt; from pydvl.parallel import JoblibParallelBackend\n&gt;&gt;&gt; parallel_backend = JoblibParallelBackend() \n&gt;&gt;&gt; with parallel_backend.executor() as executor:\n...     results = list(executor.map(lambda x: x + 1, range(3)))\n...\n&gt;&gt;&gt; results\n[1, 2, 3]\n</code></pre>"},{"location":"getting-started/advanced-usage/#map-reduce","title":"Map-reduce","text":"<p>The map-reduce interface is older and more limited in the patterns it allows us to use.</p> <p>To reproduce the previous example using MapReduceJob, we would use:</p> <pre><code>&gt;&gt;&gt; from pydvl.parallel import JoblibParallelBackend, MapReduceJob\n&gt;&gt;&gt; parallel_backend = JoblibParallelBackend() \n&gt;&gt;&gt; map_reduce_job = MapReduceJob(\n...     list(range(3)),\n...     map_func=lambda x: x[0] + 1,\n...     parallel_backend=parallel_backend,\n... )\n&gt;&gt;&gt; results = map_reduce_job()\n&gt;&gt;&gt; results\n[1, 2, 3]\n</code></pre>"},{"location":"getting-started/advanced-usage/#influence-functions","title":"Influence functions","text":"<p>Refer to Scaling influence computation for explanations about parallelization for Influence Functions.</p>"},{"location":"getting-started/advanced-usage/#getting-started-cache","title":"Caching","text":"<p>PyDVL can cache (memoize) the computation of the utility function and speed up some computations for data valuation. It is however disabled by default. When it is enabled it takes into account the data indices passed as argument and the utility function wrapped into the Utility object. This means that care must be taken when reusing the same utility function with different data, see the documentation for the caching package for more information.</p> <p>In general, caching won't play a major role in the computation of Shapley values because the probability of sampling the same subset twice, and hence needing the same utility function computation, is very low. However, it can be very useful when comparing methods that use the same utility function, or when running multiple experiments with the same data.</p> <p>pyDVL supports 3 different caching backends:</p> <ul> <li> <p>InMemoryCacheBackend:   an in-memory cache backend that uses a dictionary to store and retrieve   cached values. This is used to share cached values between threads   in a single process.</p> </li> <li> <p>DiskCacheBackend:   a disk-based cache backend that uses pickled values written to and read from disk.   This is used to share cached values between processes in a single machine.</p> </li> <li> <p>MemcachedCacheBackend:   a Memcached-based cache backend that uses pickled values written to   and read from a Memcached server. This is used to share cached values   between processes across multiple machines.</p> Memcached extras <p>The Memcached backend requires optional dependencies.  See Extras for more information.</p> </li> </ul> <p>As an example, here's how one would use the disk-based cached backend with a utility:</p> <pre><code>from pydvl.utils.caching.disk import DiskCacheBackend\nfrom pydvl.utils.utility import Utility\n\ncache_backend = DiskCacheBackend()\nu = Utility(..., cache_backend=cache_backend)\n</code></pre> <p>Please refer to the documentation and examples of each backend class for more details.</p> <p>When is the cache really necessary?</p> <p>Crucially, semi-value computations with the PermutationSampler require caching to be enabled, or they will take twice as long as the direct implementation in compute_shapley_values.</p> <p>Using the cache</p> <p>Continue reading about the cache in the documentation for the caching package.</p>"},{"location":"getting-started/advanced-usage/#setting-up-memcached","title":"Setting up the Memcached cache","text":"<p>Memcached is an in-memory key-value store accessible over the network. pyDVL can use it to cache the computation of the utility function and speed up some computations (in particular, semi-value computations with the PermutationSampler but other methods may benefit as well).</p> <p>You can either install it as a package or run it inside a docker container (the simplest). For installation instructions, refer to the Getting started section in memcached's wiki. Then you can run it with:</p> <pre><code>memcached -u user\n</code></pre> <p>To run memcached inside a container in daemon mode instead, use:</p> <pre><code>docker container run -d --rm -p 11211:11211 memcached:latest\n</code></pre>"},{"location":"getting-started/applications/","title":"Applications of data valuation","text":"<p>Data valuation methods can improve various aspects of data engineering and machine learning workflows. When applied judiciously, these methods can enhance data quality, model performance, and cost-effectiveness.</p> <p>However, the results can be inconsistent. Values have a strong dependency on the training procedure and the performance metric used. For instance, accuracy is a poor metric for imbalanced sets and this has a stark effect on data values. Some models exhibit great variance in some regimes and this again has a detrimental effect on values. See Problems of data values for more on this.</p> <p>Here we quickly enumerate the most common uses of data valuation. For a comprehensive overview, along with concrete examples, please refer to the Transferlab blog post on this topic.</p>"},{"location":"getting-started/applications/#data-engineering","title":"Data engineering","text":"<p>Some of the promising applications in data engineering include:</p> <ul> <li>Removing low-value data points to increase model performance.</li> <li>Pruning redundant samples enables more efficient training of large models.</li> <li>Active learning. Points predicted to have high-value points can be prioritized   for labeling, reducing the cost of data collection.</li> <li>Analyzing high- and low-value data to guide data collection and improve   upstream data processes. Low-value points may reveal data issues to address.</li> <li>Identify irrelevant or duplicated data when evaluating offerings from data   providers.</li> </ul>"},{"location":"getting-started/applications/#model-development","title":"Model development","text":"<p>Some of the useful applications include:</p> <ul> <li>Data attribution for interpretation and debugging: Analyzing the most or least   valuable samples for a class can reveal cases where the model relies on   confounding features instead of true signal. Investigating influential points   for misclassified examples highlights limitations to address.</li> <li>Sensitivity / robustness analysis: (Broderick et al., 2021)<sup>1</sup> shows that   removing a small fraction of highly influential data can completely flip model   conclusions. This can reveal potential issues with the modeling approach, data   collection process, or intrinsic difficulties of the problem that require   further inspection.</li> <li>Continual learning: in order to avoid forgetting when training on new data,   a subset of previously seen data is presented again. Data valuation can help   in the selection of the most valuable samples to retain.</li> </ul>"},{"location":"getting-started/applications/#attacks","title":"Attacks","text":"<p>Data valuation techniques have applications in detecting data manipulation and contamination, although the feasibility of such attacks is limited.</p> <ul> <li>Watermark removal: Points with low value on a correct validation set may be   part of a watermarking mechanism.</li> <li>Poisoning attacks: Influential points can be shifted to induce large changes   in model estimators.</li> </ul>"},{"location":"getting-started/applications/#data-markets","title":"Data markets","text":"<p>Additionally, one of the motivating applications for the whole field is that of data markets, where data valuation can be the key component to determine the price of data.</p> <p>Game-theoretic valuation methods like Shapley values can help assign fair prices, but have limitations around handling duplicates or adversarial data. Model-free methods like LAVA (Just et al., 2023)<sup>2</sup> and CRAIG are particularly well suited for this, as they use the Wasserstein distance between a vendor's data and the buyer's to determine the value of the former. </p> <p>However, this is a complex problem which can face simple practical problems like data owners not willing to disclose their data for valuation, even to a broker.</p> <ol> <li> <p>Broderick, T., Giordano, R., Meager, R., 2021. An Automatic Finite-Sample Robustness Metric: When Can Dropping a Little Data Make a Big Difference? \u21a9</p> </li> <li> <p>Just, H.A., Kang, F., Wang, T., Zeng, Y., Ko, M., Jin, M., Jia, R., 2023. LAVA: Data Valuation without Pre-Specified Learning Algorithms. Presented at the The Eleventh International Conference on Learning Representations (ICLR 2023).\u00a0\u21a9</p> </li> </ol>"},{"location":"getting-started/benchmarking/","title":"Benchmarking tasks","text":"<p>Because the magnitudes of values or influences from different algorithms, or datasets, are not comparable to each other, evaluation of the methods is typically done with downstream tasks.</p>"},{"location":"getting-started/benchmarking/#benchmarking-valuation-methods","title":"Benchmarking valuation methods","text":"<p>Data valuation is particularly useful for data selection, pruning and inspection in general. For this reason, the most common benchmarks are data removal and noisy label detection.</p>"},{"location":"getting-started/benchmarking/#high-value-point-removal","title":"High-value point removal","text":"<p>After computing the values for all data in \\(T = \\{ \\mathbf{z}_i : i = 1, \\ldots, n \\}\\), the set is sorted by decreasing value. We denote by \\(T_{[i :]}\\) the sorted sequence of points \\((\\mathbf{z}_i, \\mathbf{z}_{i + 1}, \\ldots, \\mathbf{z}_n)\\) for \\(1 \\leqslant i \\leqslant n\\). Now train successively \\(f_{T [i :]}\\) and compute its accuracy \\(a_{T_{[i :]}} (D_{\\operatorname{test}})\\) on the held-out test set, then plot all numbers. By using \\(D_{\\operatorname{test}}\\) one approximates the expected accuracy drop on unseen data. Because the points removed have a high value, one expects performance to drop visibly wrt. a random baseline.</p>"},{"location":"getting-started/benchmarking/#low-value-point-removal","title":"Low-value point removal","text":"<p>The complementary experiment removes data in increasing order, with the lowest valued points first. Here one expects performance to increase relatively to randomly removing points before training. Additionally, every real dataset will include slightly out-of-distribution points, so one should also expect an absolute increase in performance when some of the lowest valued points are removed.</p>"},{"location":"getting-started/benchmarking/#value-transfer","title":"Value transfer","text":"<p>This experiment explores the extent to which data values computed with one (cheap) model can be transferred to another (potentially more complex) one. Different classifiers are used as a source to calculate data values. These values are then used in the point removal tasks described above, but using a different (target) model for evaluation of the accuracies \\(a_{T [i :]}\\). A multi-layer perceptron is added for evaluation as well.</p>"},{"location":"getting-started/benchmarking/#noisy-label-detection","title":"Noisy label detection","text":"<p>This experiment tests the ability of a method to detect mislabeled instances in the data. A fixed fraction \\(\\alpha\\) of the training data are picked at random and their labels flipped. Data values are computed, then the \\(\\alpha\\)-fraction of lowest-valued points are selected, and the overlap with the subset of flipped points is computed. This synthetic experiment is however hard to put into practical use, since the fraction \\(\\alpha\\) is of course unknown in practice.</p>"},{"location":"getting-started/benchmarking/#rank-stability","title":"Rank stability","text":"<p>Introduced in [@wang_data_2022], one can  look at how stable the top \\(k\\)% of the values is across runs. Rank stability of a method is necessary but not sufficient for good results. Ideally one wants to identify high-value points reliably (good precision and recall) and consistently (good rank stability).</p>"},{"location":"getting-started/benchmarking/#benchmarking-influence-function-methods","title":"Benchmarking Influence function methods","text":"<p>Todo</p> <p>This section is basically a stub</p> <p>Although in principle one can compute the average influence over the test set and run the same tasks as above, because influences are computed for each pair of training and test sample, they typically require different experiments to compare their efficacy.</p>"},{"location":"getting-started/benchmarking/#approximation-quality","title":"Approximation quality","text":"<p>The biggest difficulty when computing influences is the approximation of the inverse Hessian-vector product. For this reason one often sees in the literature the quality of the approximation to LOO as an indicator of performance, the exact Influence Function being a first order approximation to it. However, as shown by (Bae et al., 2022)<sup>1</sup>, the different approximation errors ensuing for lack of convexity, approximate Hessian-vector products and so on, lead to this being a poor benchmark overall.</p>"},{"location":"getting-started/benchmarking/#data-re-labelling","title":"Data re-labelling","text":"<p>(Kong et al., 2022)<sup>2</sup> introduce a method using IFs to re-label harmful training samples in order to improve accuracy. One can then take the obtained improvement as a measure of the quality of the IF method.</p>"},{"location":"getting-started/benchmarking/#post-hoc-fairness-adjustment","title":"Post-hoc fairness adjustment","text":"<p>Introduced in [@...], the idea is to compute influences over a carefully selected fair set, and using them to re-weight the training data.</p> <ol> <li> <p>Bae, J., Ng, N., Lo, A., Ghassemi, M., Grosse, R.B., 2022. If Influence Functions are the Answer, Then What is the Question?, in: Advances in Neural Information Processing Systems. Presented at the NeurIPS 2022, pp. 17953\u201317967.\u00a0\u21a9</p> </li> <li> <p>Kong, S., Shen, Y., Huang, L., 2022. Resolving Training Biases via Influence-based Data Relabeling. Presented at the International Conference on Learning Representations (ICLR 2022).\u00a0\u21a9</p> </li> </ol>"},{"location":"getting-started/first-steps/","title":"First steps","text":"<p>Warning</p> <p>Make sure you have read Getting started before using the library.  In particular read about which extra dependencies you may need.</p>"},{"location":"getting-started/first-steps/#main-concepts","title":"Main concepts","text":"<p>pyDVL aims to be a repository of production-ready, reference implementations of algorithms for data valuation and influence functions. Even though we only briefly introduce key concepts in the documentation, the following sections  should be enough to get you started.</p> <ul> <li>Basics of data valuation for key objects and usage patterns for Shapley value   computation and related methods.</li> <li>Computing Influence Values for instructions on how to compute influence functions.</li> </ul>"},{"location":"getting-started/first-steps/#running-the-examples","title":"Running the examples","text":"<p>If you are somewhat familiar with the concepts of data valuation, you can start by browsing our worked-out examples illustrating pyDVL's capabilities either:</p> <ul> <li>In the examples under Basics of data valuation and Computing Influence Values.</li> <li>Using binder notebooks, deployed from each   example's page.</li> <li>Locally, by starting a jupyter server at the root of the project. You will   have to install jupyter first manually since it's not a dependency of the   library.</li> </ul>"},{"location":"getting-started/first-steps/#advanced-usage","title":"Advanced usage","text":"<p>Refer to the Advanced usage page for explanations on how to enable and use parallelization and caching.</p>"},{"location":"getting-started/glossary/","title":"Glossary","text":"<p>This glossary is meant to provide only brief explanations of each term, helping to clarify the concepts and techniques used in the library. For more detailed information, please refer to the relevant literature or resources.</p> <p>Warning</p> <p>This glossary is still a work in progress. Pull requests are welcome!</p> <p>Terms in data valuation and influence functions:</p>"},{"location":"getting-started/glossary/#arnoldi-method","title":"Arnoldi Method","text":"<p>The Arnoldi method approximately computes eigenvalue, eigenvector pairs of a symmetric matrix. For influence functions, it is used to approximate the iHVP. Introduced by (Schioppa et al., 2022)<sup>1</sup> in the context of influence functions.</p> <ul> <li>Implementation (torch)     </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#block-conjugate-gradient","title":"Block Conjugate Gradient","text":"<p>A blocked version of CG, which solves several linear systems simultaneously. For Influence Functions, it is used to approximate the iHVP.</p> <ul> <li>Implementation (torch)    </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#class-wise-shapley","title":"Class-wise Shapley","text":"<p>Class-wise Shapley is a Shapley valuation method which introduces a utility function that balances in-class, and out-of-class accuracy, with the goal of favoring points that improve the model's performance on the class they belong to. It is estimated to be particularly useful in imbalanced datasets, but more research is needed to confirm this. Introduced by (Schoch et al., 2022)<sup>2</sup>.</p> <ul> <li>Implementation    </li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#conjugate-gradient","title":"Conjugate Gradient","text":"<p>CG is an algorithm for solving linear systems with a symmetric and positive-definite coefficient matrix. For Influence Functions, it is used to approximate the iHVP.</p> <ul> <li>Implementation (torch) </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#data-utility-learning","title":"Data Utility Learning","text":"<p>Data Utility Learning is a method that uses an ML model to learn the utility function. Essentially, it learns to predict the performance of a model when trained on a given set of indices from the dataset. The cost of training this model is quickly amortized by avoiding costly re-evaluations of the original utility. Introduced by (Wang et al., 2022)<sup>3</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#eigenvalue-corrected-kronecker-factored-approximate-curvature","title":"Eigenvalue-corrected Kronecker-Factored Approximate Curvature","text":"<p>EKFAC builds on K-FAC by correcting for the approximation errors in the eigenvalues of the blocks of the Kronecker-factored approximate curvature matrix. This correction aims to refine the accuracy of natural gradient approximations, thus potentially offering better training efficiency and stability in neural networks.</p> <ul> <li>Implementation (torch)    </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#group-testing","title":"Group Testing","text":"<p>Group Testing is a strategy for identifying characteristics within groups of items efficiently, by testing groups rather than individuals to quickly narrow down the search for items with specific properties. Introduced into data valuation by (Jia et al., 2019)<sup>4</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#influence-function","title":"Influence Function","text":"<p>The Influence Function measures the impact of a single data point on a statistical estimator. In machine learning, it's used to understand how much a particular data point affects the model's prediction. Introduced into data valuation by (Koh and Liang, 2017)<sup>5</sup>.</p> <ul> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#inverse-hessian-vector-product","title":"Inverse Hessian-vector product","text":"<p>iHVP is the operation of calculating the product of the inverse Hessian matrix of a function and a vector, without explicitly constructing nor inverting the full Hessian matrix first. This is essential for influence function computation.</p>"},{"location":"getting-started/glossary/#kronecker-factored-approximate-curvature","title":"Kronecker-Factored Approximate Curvature","text":"<p>K-FAC is an optimization technique that approximates the Fisher Information matrix's inverse efficiently. It uses the Kronecker product to factor the matrix, significantly speeding up the computation of natural gradient updates and potentially improving training efficiency.</p>"},{"location":"getting-started/glossary/#least-core","title":"Least Core","text":"<p>The Least Core is a solution concept in cooperative game theory, referring to the smallest set of payoffs to players that cannot be improved upon by any coalition, ensuring stability in the allocation of value. In data valuation, it implies solving a linear and a quadratic system whose constraints are determined by the evaluations of the utility function on every subset of the training data. Introduced as data valuation method by (Yan and Procaccia, 2021)<sup>6</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#linear-time-stochastic-second-order-algorithm","title":"Linear-time Stochastic Second-order Algorithm","text":"<p>LiSSA is an efficient algorithm for approximating the inverse Hessian-vector product, enabling faster computations in large-scale machine learning problems, particularly for second-order optimization. For Influence Functions, it is used to approximate the iHVP. Introduced by (Agarwal et al., 2017)<sup>7</sup>.</p> <ul> <li>Implementation (torch)    </li> <li>Documentation (torch)    </li> </ul>"},{"location":"getting-started/glossary/#leave-one-out","title":"Leave-One-Out","text":"<p>LOO in the context of data valuation refers to the process of evaluating the impact of removing individual data points on the model's performance. The value of a training point is defined as the marginal change in the model's performance when that point is removed from the training set.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#maximum-sample-reuse","title":"Maximum Sample Reuse","text":"<p>MSR is a sampling method for data valuation that updates the value of every data point in one sample. This method can achieve much faster convergence. Introduced by (Wang and Jia, 2023)<sup>8</sup></p> <ul> <li>Implementation</li> </ul>"},{"location":"getting-started/glossary/#monte-carlo-least-core","title":"Monte Carlo Least Core","text":"<p>MCLC is a variation of the Least Core that uses a reduced amount of constraints, sampled randomly from the powerset of the training data. Introduced by (Yan and Procaccia, 2021)<sup>6</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#monte-carlo-shapley","title":"Monte Carlo Shapley","text":"<p>MCS estimates the Shapley Value using a Monte Carlo approximation to the sum over subsets of the training set. This reduces computation to polynomial time at the cost of accuracy, but this loss is typically irrelevant for downstream applications in ML. Introduced into data valuation by (Ghorbani and Zou, 2019)<sup>9</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#nystrom-low-rank-approximation","title":"Nystr\u00f6m Low-Rank Approximation","text":"<p>The Nystr\u00f6m approximation computes a low-rank approximation to a symmetric positive-definite matrix via random projections. For influence functions,  it is used to approximate the iHVP. Introduced as sketch and solve algorithm in (Hataya and Yamada, 2023)<sup>10</sup>, and as preconditioner for PCG in  (Frangella et al., 2023)<sup>11</sup>.</p> <ul> <li>Implementation Sketch-and-Solve (torch)    </li> <li>Documentation Sketch-and-Solve (torch)</li> <li>Implementation Preconditioner (torch)    </li> </ul>"},{"location":"getting-started/glossary/#point-removal-task","title":"Point removal task","text":"<p>A task in data valuation where the quality of a valuation method is measured through the impact of incrementally removing data points on the model's performance, where the points are removed in order of their value. See</p> <ul> <li>Benchmarking tasks</li> </ul>"},{"location":"getting-started/glossary/#preconditioned-block-conjugate-gradient","title":"Preconditioned Block Conjugate Gradient","text":"<p>A blocked version of PCG, which solves  several linear systems simultaneously. For Influence Functions, it is used to approximate the iHVP.</p> <ul> <li>Implementation CG (torch)    </li> <li>Implementation Preconditioner (torch)    </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#preconditioned-conjugate-gradient","title":"Preconditioned Conjugate Gradient","text":"<p>A preconditioned version of CG for improved convergence, depending on the characteristics of the matrix and the preconditioner. For Influence Functions, it is used to approximate the iHVP.</p> <ul> <li>Implementation CG (torch)    </li> <li>Implementation Preconditioner (torch)    </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#shapley-value","title":"Shapley Value","text":"<p>Shapley Value is a concept from cooperative game theory that allocates payouts to players based on their contribution to the total payoff. In data valuation, players are data points. The method assigns a value to each data point based on a weighted average of its marginal contributions to the model's performance  when trained on each subset of the training set. This requires \\(\\mathcal{O}(2^{n-1})\\) re-trainings of the model, which is infeasible for even trivial data set sizes, so one resorts to approximations like TMCS. Introduced into data valuation by (Ghorbani and Zou, 2019)<sup>9</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#truncated-monte-carlo-shapley","title":"Truncated Monte Carlo Shapley","text":"<p>TMCS is an efficient approach to estimating the Shapley Value using a truncated version of the Monte Carlo method, reducing computation time while maintaining accuracy in large datasets. Introduced by (Ghorbani and Zou, 2019)<sup>9</sup>.</p> <ul> <li>Implementation    </li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#weighted-accuracy-drop","title":"Weighted Accuracy Drop","text":"<p>WAD is a metric to evaluate the impact of sequentially removing data points on the performance of a machine learning model, weighted by their rank, i.e. by the time at which they were removed. Introduced by (Schoch et al., 2022)<sup>2</sup>.</p>"},{"location":"getting-started/glossary/#other-terms","title":"Other terms","text":""},{"location":"getting-started/glossary/#coefficient-of-variation","title":"Coefficient of Variation","text":"<p>CV is a statistical measure of the dispersion of data points in a data series around the mean, expressed as a percentage. It's used to compare the degree of variation from one data series to another, even if the means are drastically different.</p>"},{"location":"getting-started/glossary/#constraint-satisfaction-problem","title":"Constraint Satisfaction Problem","text":"<p>A CSP involves finding values for variables within specified constraints or conditions, commonly used in scheduling, planning, and design problems where solutions must satisfy a set of restrictions.</p>"},{"location":"getting-started/glossary/#out-of-bag","title":"Out-of-Bag","text":"<p>OOB refers to data samples in an ensemble learning context (like random forests) that are not selected for training a specific model within the ensemble. These OOB samples are used as a validation set to estimate the model's accuracy, providing a convenient internal cross-validation mechanism.</p>"},{"location":"getting-started/glossary/#machine-learning-reproducibility-challenge","title":"Machine Learning Reproducibility Challenge","text":"<p>The MLRC is an initiative that encourages the verification and replication of machine learning research findings, promoting transparency and reliability in the field. Papers are published in Transactions on Machine Learning Research (TMLR).</p> <ol> <li> <p>Schioppa, A., Zablotskaia, P., Vilar, D., Sokolov, A., 2022. Scaling Up Influence Functions. Proc. AAAI Conf. Artif. Intell. 36, 8179\u20138186. https://doi.org/10.1609/aaai.v36i8.20791 \u21a9</p> </li> <li> <p>Schoch, S., Xu, H., Ji, Y., 2022. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification, in: Proc. Of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). Presented at the Advances in Neural Information Processing Systems (NeurIPS 2022).\u00a0\u21a9\u21a9</p> </li> <li> <p>Wang, T., Yang, Y., Jia, R., 2022. Improving Cooperative Game Theory-based Data Valuation via Data Utility Learning. Presented at the International Conference on Learning Representations (ICLR 2022). Workshop on Socially Responsible Machine Learning, arXiv. https://doi.org/10.48550/arXiv.2107.06336 \u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Gurel, N.M., Li, B., Zhang, C., Spanos, C., Song, D., 2019. Efficient task-specific data valuation for nearest neighbor algorithms. Proc. VLDB Endow. 12, 1610\u20131623. https://doi.org/10.14778/3342263.3342637 \u21a9</p> </li> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions, in: Proceedings of the 34th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 1885\u20131894.\u00a0\u21a9</p> </li> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9\u21a9</p> </li> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. JMLR 18, 1\u201340.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T., Jia, R., 2023. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning, in: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 6388\u20136421.\u00a0\u21a9</p> </li> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9\u21a9\u21a9</p> </li> <li> <p>Hataya, R., Yamada, M., 2023. Nystr\u00f6m Method for Accurate and Scalable Implicit Differentiation, in: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 4643\u20134654.\u00a0\u21a9</p> </li> <li> <p>Frangella, Z., Tropp, J.A., Udell, M., 2023. Randomized Nystr\u00f6m Preconditioning. SIAM J. Matrix Anal. Appl. 44, 718\u2013752. https://doi.org/10.1137/21M1466244 \u21a9</p> </li> </ol>"},{"location":"getting-started/methods/","title":"Methods","text":"<p>We currently implement the following methods:</p>"},{"location":"getting-started/methods/#data-valuation","title":"Data valuation","text":"<ul> <li> <p>LOO.</p> </li> <li> <p>Permutation Shapley   (also called ApproxShapley) (Castro et al., 2009)<sup>1</sup>.</p> </li> <li> <p>TMCS   (Ghorbani and Zou, 2019)<sup>2</sup>.</p> </li> <li> <p>Data Banzhaf   [@wang_data_2022].</p> </li> <li> <p>Beta Shapley   (Kwon and Zou, 2022)<sup>3</sup>.</p> </li> <li> <p>CS-Shapley   (Schoch et al., 2022)<sup>4</sup>.</p> </li> <li> <p>Least Core   (Yan and Procaccia, 2021)<sup>5</sup>.</p> </li> <li> <p>Owen Sampling   (Okhrati and Lipani, 2021)<sup>6</sup>.</p> </li> <li> <p>Data Utility Learning   (Wang et al., 2022)<sup>7</sup>.</p> </li> <li> <p>kNN-Shapley   (Jia et al., 2019)<sup>8</sup>.</p> </li> <li> <p>Group Testing   (Jia et al., 2019)<sup>9</sup></p> </li> <li> <p>Data-OOB   (Kwon and Zou, 2023)<sup>10</sup>.</p> </li> </ul>"},{"location":"getting-started/methods/#influence-functions","title":"Influence functions","text":"<ul> <li> <p>CG Influence.   (Koh and Liang, 2017)<sup>11</sup>.</p> </li> <li> <p>Direct Influence   (Koh and Liang, 2017)<sup>11</sup>.</p> </li> <li> <p>LiSSA   (Agarwal et al., 2017)<sup>12</sup>.</p> </li> <li> <p>Arnoldi Influence   (Schioppa et al., 2022)<sup>13</sup>.</p> </li> <li> <p>EKFAC Influence   (George et al., 2018; Martens and Grosse, 2015)<sup>14</sup><sup>15</sup>.</p> </li> <li> <p>Nystr\u00f6m Influence, based   on the ideas in (Hataya and Yamada, 2023)<sup>16</sup> for bi-level optimization.</p> </li> </ul> <ol> <li> <p>Castro, J., G\u00f3mez, D., Tejada, J., 2009. Polynomial calculation of the Shapley value based on sampling. Computers   &amp; Operations Research, Selected papers presented at the Tenth International Symposium on Locational Decisions (ISOLDE X) 36, 1726\u20131730. https://doi.org/10.1016/j.cor.2008.04.004 \u21a9</p> </li> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning, in: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022,. Presented at the AISTATS 2022, PMLR.\u00a0\u21a9</p> </li> <li> <p>Schoch, S., Xu, H., Ji, Y., 2022. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification, in: Proc. Of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). Presented at the Advances in Neural Information Processing Systems (NeurIPS 2022).\u00a0\u21a9</p> </li> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9</p> </li> <li> <p>Okhrati, R., Lipani, A., 2021. A Multilinear Sampling Algorithm to Estimate Shapley Values, in: 2020 25th International Conference on Pattern Recognition (ICPR). Presented at the 2020 25th International Conference on Pattern Recognition (ICPR), IEEE, pp. 7992\u20137999. https://doi.org/10.1109/ICPR48806.2021.9412511 \u21a9</p> </li> <li> <p>Wang, T., Yang, Y., Jia, R., 2022. Improving Cooperative Game Theory-based Data Valuation via Data Utility Learning. Presented at the International Conference on Learning Representations (ICLR 2022). Workshop on Socially Responsible Machine Learning, arXiv. https://doi.org/10.48550/arXiv.2107.06336 \u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Gurel, N.M., Li, B., Zhang, C., Spanos, C., Song, D., 2019. Efficient task-specific data valuation for nearest neighbor algorithms. Proc. VLDB Endow. 12, 1610\u20131623. https://doi.org/10.14778/3342263.3342637 \u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Hynes, N., G\u00fcrel, N.M., Li, B., Zhang, C., Song, D., Spanos, C.J., 2019. Towards Efficient Data Valuation Based on the Shapley Value, in: Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics (AISTATS), PMLR, pp. 1167\u20131176.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2023. Data-OOB: Out-of-bag Estimate as a Simple and Efficient Data Value, in: Proceedings of the 40th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 18135\u201318152.\u00a0\u21a9</p> </li> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions, in: Proceedings of the 34th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 1885\u20131894.\u00a0\u21a9\u21a9</p> </li> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. JMLR 18, 1\u201340.\u00a0\u21a9</p> </li> <li> <p>Schioppa, A., Zablotskaia, P., Vilar, D., Sokolov, A., 2022. Scaling Up Influence Functions. Proc. AAAI Conf. Artif. Intell. 36, 8179\u20138186. https://doi.org/10.1609/aaai.v36i8.20791 \u21a9</p> </li> <li> <p>George, T., Laurent, C., Bouthillier, X., Ballas, N., Vincent, P., 2018. Fast Approximate Natural Gradient Descent in a Kronecker Factored Eigenbasis, in: Advances in Neural Information Processing Systems. Curran Associates, Inc.\u00a0\u21a9</p> </li> <li> <p>Martens, J., Grosse, R., 2015. Optimizing Neural Networks with Kronecker-factored Approximate Curvature, in: Proceedings of the 32nd International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 2408\u20132417.\u00a0\u21a9</p> </li> <li> <p>Hataya, R., Yamada, M., 2023. Nystr\u00f6m Method for Accurate and Scalable Implicit Differentiation, in: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 4643\u20134654.\u00a0\u21a9</p> </li> </ol>"},{"location":"influence/","title":"The influence function","text":""},{"location":"influence/#the-influence-function","title":"The influence function","text":"<p>Warning</p> <p>The code in the package pydvl.influence is experimental. Package structure and basic API are bound to change before v1.0.0</p> <p>The influence function (IF) is a method to quantify the effect (influence) that each training point has on the parameters of a model, and by extension on any function thereof. In particular, it allows to estimate how much each training sample affects the error on a test point, making the IF useful for understanding and debugging models.</p> <p>Alas, the influence function relies on some assumptions that can make their application difficult. Yet another drawback is that they require the computation of the inverse of the Hessian of the model wrt. its parameters, which is intractable for large models like deep neural networks. Much of the recent research tackles this issue using approximations, like a Neuman series  (Agarwal et al., 2017)<sup>1</sup>, with the most successful solution using a low-rank approximation that iteratively finds increasing eigenspaces of the Hessian  (Schioppa et al., 2022)<sup>2</sup>.</p> <p>pyDVL implements several methods for the efficient computation of the IF for machine learning. In the examples we document some of the difficulties that can arise when using the IF.</p>"},{"location":"influence/#construction","title":"Construction","text":"<p>First introduced in the context of robust statistics in (Hampel, 1974)<sup>3</sup>, the IF was popularized in the context of machine learning in  (Koh and Liang, 2017)<sup>4</sup>.</p> <p>Following their formulation, consider an input space \\(\\mathcal{X}\\) (e.g. images) and an output space \\(\\mathcal{Y}\\) (e.g. labels). Let's take \\(z_i = (x_i, y_i)\\), for \\(i \\in  \\{1,...,n\\}\\) to be the \\(i\\)-th training point, and \\(\\theta\\) to be the (potentially highly) multi-dimensional parameters of a model (e.g. \\(\\theta\\) is a big array with all of a neural network's parameters, including biases and/or dropout rates). We will denote with \\(L(z, \\theta)\\) the loss of the model for point \\(z\\) when the parameters are \\(\\theta.\\)</p> <p>To train a model, we typically minimize the loss over all \\(z_i\\), i.e. the optimal parameters are</p> \\[\\hat{\\theta} = \\arg \\min_\\theta \\sum_{i=1}^n L(z_i, \\theta).\\] <p>In practice, lack of convexity means that one doesn't really obtain the minimizer of the loss, and the training is stopped when the validation loss stops decreasing.</p> <p>For notational convenience, let's define</p> \\[\\hat{\\theta}_{-z} = \\arg \\min_\\theta \\sum_{z_i \\ne z} L(z_i, \\theta), \\] <p>i.e. \\(\\hat{\\theta}_{-z}\\) are the model parameters that minimize the total loss when \\(z\\) is not in the training dataset.</p> <p>In order to compute the impact of each training point on the model, we would need to calculate \\(\\hat{\\theta}_{-z}\\) for each \\(z\\) in the training dataset, thus re-training the model at least ~\\(n\\) times (more if model training is stochastic). This is computationally very expensive, especially for big neural networks. To circumvent this problem, we can just calculate a first order approximation of \\(\\hat{\\theta}\\). This can be done through single backpropagation and without re-training the full model.</p> <p>pyDVL supports two ways of computing the empirical influence function, namely up-weighting of samples and perturbation influences.</p>"},{"location":"influence/#approximating-the-influence-of-a-point","title":"Approximating the influence of a point","text":"<p>Let's define</p> \\[\\hat{\\theta}_{\\epsilon, z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) + \\epsilon L(z, \\theta), \\] <p>which is the optimal \\(\\hat{\\theta}\\) when we up-weight \\(z\\) by an amount \\(\\epsilon \\gt 0\\).</p> <p>From a classical result (a simple derivation is available in Appendix A of  (Koh and Liang, 2017)<sup>4</sup>), we know that:</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta L(z, \\hat{\\theta}), \\] <p>where \\(H_{\\hat{\\theta}} = \\frac{1}{n} \\sum_{i=1}^n \\nabla_\\theta^2 L(z_i, \\hat{\\theta})\\) is the Hessian of \\(L\\). These quantities are also knows as influence factors.</p> <p>Importantly, notice that this expression is only valid when \\(\\hat{\\theta}\\) is a minimum of \\(L\\), or otherwise \\(H_{\\hat{\\theta}}\\) cannot be inverted! At the same time, in machine learning full convergence is rarely achieved, so direct Hessian inversion is not possible. Approximations need to be developed that circumvent the problem of inverting the Hessian of the model in all those (frequent) cases where it is not positive definite.</p> <p>The influence of training point \\(z\\) on test point \\(z_{\\text{test}}\\) is defined as:</p> \\[\\mathcal{I}(z, z_{\\text{test}}) =  L(z_{\\text{test}}, \\hat{\\theta}_{-z}) - L(z_{\\text{test}}, \\hat{\\theta}). \\] <p>Notice that \\(\\mathcal{I}\\) is higher for points \\(z\\) which positively impact the model score, since the loss is higher when they are excluded from training. In practice, one needs to rely on the following infinitesimal approximation:</p> \\[\\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\frac{d L(z_{\\text{test}}, \\hat{\\theta}_{\\epsilon, z})}{d \\epsilon} \\Big|_{\\epsilon=0} \\] <p>Using the chain rule and the results calculated above, we get:</p> \\[\\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_\\theta L(z, \\hat{\\theta}) \\] <p>All the resulting factors are gradients of the loss wrt. the model parameters \\(\\hat{\\theta}\\). This can be easily computed through one or more backpropagation passes.</p>"},{"location":"influence/#perturbation-definition-of-the-influence-score","title":"Perturbation definition of the influence score","text":"<p>How would the loss of the model change if, instead of up-weighting an individual point \\(z\\), we were to up-weight only a single feature of that point? Given \\(z = (x, y)\\), we can define \\(z_{\\delta} = (x+\\delta, y)\\), where \\(\\delta\\) is a vector of zeros except for a 1 in the position of the feature we want to up-weight. In order to approximate the effect of modifying a single feature of a single point on the model score we can define</p> \\[\\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_{i}, \\theta) + \\epsilon L(z_{\\delta}, \\theta) - \\epsilon L(z, \\theta), \\] <p>Similarly to what was done above, we up-weight point \\(z_{\\delta}\\), but then we also remove the up-weighting for all the features that are not modified by \\(\\delta\\). From the calculations in the previous section, it is then easy to see that</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta \\Big( L(z_{\\delta}, \\hat{\\theta}) - L(z, \\hat{\\theta}) \\Big) \\] <p>and if the feature space is continuous and as \\(\\delta \\to 0\\) we can write</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\ \\nabla_x \\nabla_\\theta L(z, \\hat{\\theta}) \\delta + \\mathcal{o}(\\delta) \\] <p>The influence of each feature of \\(z\\) on the loss of the model can therefore be estimated through the following quantity:</p> \\[\\mathcal{I}_{pert}(z, z_{\\text{test}}) = - \\lim_{\\delta \\to 0} \\ \\frac{1}{\\delta} \\frac{d L(z_{\\text{test}}, \\hat{\\theta}_{\\epsilon, \\ z_{\\delta}, \\ -z})}{d \\epsilon} \\Big|_{\\epsilon=0} \\] <p>which, using the chain rule and the results calculated above, is equal to</p> \\[\\mathcal{I}_{pert}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_x \\nabla_\\theta L(z, \\hat{\\theta}) \\] <p>The perturbation definition of the influence score is not straightforward to understand, but it has a simple interpretation: it tells how much the loss of the model changes when a certain feature of point z is up-weighted. A positive perturbation influence score indicates that the feature might have a positive effect on the accuracy of the model.</p> <p>It is worth noting that the perturbation influence score is a very rough estimate of the impact of a point on the models loss and it is subject to large approximation errors. It can nonetheless be used to build training-set attacks, as done in (Koh and Liang, 2017)<sup>4</sup>.</p>"},{"location":"influence/#computation","title":"Computation","text":"<p>The main abstraction of the library for influence calculation is InfluenceFunctionModel.  On implementations of this abstraction, you can call the method <code>influences</code> to compute influences. </p> <p>pyDVL provides implementations to use with pytorch model in pydvl.influence.torch. For detailed information  on available implementations see the documentation in InfluenceFunctionModel.</p> <p>Given a pre-trained pytorch model and a loss, a basic example would look like</p> <p><pre><code>from torch.utils.data import DataLoader\nfrom pydvl.influence.torch import DirectInfluence\n\ntraining_data_loader = DataLoader(...)\ninfl_model = DirectInfluence(model, loss)\ninfl_model = infl_model.fit(training_data_loader)\n\ninfluences = infl_model.influences(x_test, y_test, x, y)\n</code></pre> for batches \\(z_{\\text{test}} = (x_{\\text{test}}, y_{\\text{test}})\\) and \\(z = (x, y)\\) of data. The result is a tensor with one row per test point in  \\(z_{\\text{test}}\\) and one column per point in \\(z\\).  Thus, each entry \\((i, j)\\) represents the influence of training point \\(z[j]\\) on test point \\(z_{\\text{test}}[i]\\).</p> <p>Warning</p> <p>Compared to the mathematical definitions above, we switch the ordering of \\(z\\) and \\(z_{\\text{test}}\\), in order to make the input ordering consistent with the dimensions of the resulting tensor. More concrete if the first dimension of \\(z_{\\text{test}}\\) is \\(N\\) and that of \\(z\\), the resulting tensor is of shape \\(N \\times M\\)</p> <p>A large positive influence indicates that training point \\(j\\) tends to improve the performance of the model on test point \\(i\\), and vice versa, a large negative influence indicates that training point \\(j\\) tends to worsen the performance of the model on test point \\(i\\).</p>"},{"location":"influence/#hessian-regularization","title":"Hessian regularization","text":"<p>Additionally, and as discussed in the introduction, in machine learning training rarely converges to a global minimum of the loss. Despite good apparent convergence, \\(\\hat{\\theta}\\) might be located in a region with flat curvature or close to a saddle point. In particular, the Hessian might have vanishing eigenvalues making its direct inversion impossible. Certain methods, such as the Arnoldi method are robust against these problems, but most are not.</p> <p>To circumvent this problem, many approximate methods can be implemented. The simplest adds a small hessian perturbation term, i.e. \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\), with \\(\\mathbb{I}\\) being the identity matrix. </p> <pre><code>from torch.utils.data import DataLoader\nfrom pydvl.influence.torch import DirectInfluence\n\ntraining_data_loader = DataLoader(...)\ninfl_model = DirectInfluence(model, loss, hessian_regularization=0.01)\ninfl_model = infl_model.fit(training_data_loader)\n</code></pre> <p>This standard trick ensures that the eigenvalues of \\(H_{\\hat{\\theta}}\\) are bounded away from zero and therefore the matrix is invertible. In order for this regularization not to corrupt the outcome too much, the parameter \\(\\lambda\\) should be as small as possible while still allowing a reliable inversion of \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\).</p>"},{"location":"influence/#perturbation-influences","title":"Perturbation influences","text":"<p>The method of empirical influence computation can be selected with the parameter <code>mode</code>:</p> <p><pre><code>from pydvl.influence import InfluenceMode\n\ninfluences = infl_model.influences(x_test, y_test, x, y,\n                                   mode=InfluenceMode.Perturbation)\n</code></pre> The result is a tensor with at least three dimensions. The first two dimensions are the same as in the case of <code>mode=InfluenceMode.Up</code> case, i.e. one row per test point and one column per training point. The remaining dimensions are the same as the number of input features in the data. Therefore, each entry in the tensor represents the influence of each feature of each training point on each test point.</p>"},{"location":"influence/#influence-factors","title":"Influence factors","text":"<p>The influence factors(refer to the previous section for a definition) are typically the most computationally demanding part of influence calculation. They can be obtained via calling the <code>influence_factors</code> method, saved, and later used  for influence calculation on different subsets of the training dataset.</p> <pre><code>influence_factors = infl_model.influence_factors(x_test, y_test)\ninfluences = infl_model.influences_from_factors(influence_factors, x, y)\n</code></pre> <ol> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. JMLR 18, 1\u201340.\u00a0\u21a9</p> </li> <li> <p>Schioppa, A., Zablotskaia, P., Vilar, D., Sokolov, A., 2022. Scaling Up Influence Functions. Proc. AAAI Conf. Artif. Intell. 36, 8179\u20138186. https://doi.org/10.1609/aaai.v36i8.20791 \u21a9</p> </li> <li> <p>Hampel, F.R., 1974. The Influence Curve and Its Role in Robust Estimation. J. Am. Stat. Assoc. 69, 383\u2013393. https://doi.org/10.2307/2285666 \u21a9</p> </li> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions, in: Proceedings of the 34th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 1885\u20131894.\u00a0\u21a9\u21a9\u21a9</p> </li> </ol>"},{"location":"influence/influence_function_model/","title":"Influence Function Model","text":"<p>In almost every practical application it is not possible to construct, even less invert the complete Hessian in memory. pyDVL offers several implementations of  the interface InfluenceFunctionModel ,  which do not compute the full Hessian (in contrast to DirectInfluence ).</p>"},{"location":"influence/influence_function_model/#conjugate-gradient","title":"Conjugate Gradient","text":"<p>This classical procedure for solving linear systems of equations is an iterative method that does not require the explicit inversion of the Hessian. Instead, it only requires the calculation of Hessian-vector products, making it a good choice for large datasets or models with many parameters. It is nevertheless much slower to converge than the direct inversion method and not as accurate.</p> <p>More info on the theory of conjugate gradient can be found on Wikipedia, or in text books such as (Trefethen and Bau, 1997, Lecture 38)<sup>1</sup>.</p> <p>pyDVL also implements a stable block variant of the conjugate  gradient method, defined in (Ji and Li, 2017)<sup>2</sup>, which solves several right hand sides simultaneously.</p> <p>Optionally, the user can provide a pre-conditioner to improve convergence, such  as a Jacobi pre-conditioner , which is a simple diagonal pre-conditioner  based on Hutchinson's diagonal estimator (Bekas et al., 2007)<sup>3</sup>, or a Nystr\u00f6m approximation based pre-conditioner ,  described in (Frangella et al., 2023)<sup>4</sup>. </p> <pre><code>from pydvl.influence.torch import CgInfluence\nfrom pydvl.influence.torch.pre_conditioner import NystroemPreConditioner\n\nif_model = CgInfluence(\n    model,\n    loss,\n    hessian_regularization=0.0,\n    rtol=1e-7,\n    atol=1e-7,\n    maxiter=None,\n    use_block_cg=True,\n    pre_conditioner=NystroemPreConditioner(rank=10)\n)\nif_model.fit(train_loader)\n</code></pre> <p>The additional optional parameters <code>rtol</code>, <code>atol</code>, <code>maxiter</code>, <code>use_block_cg</code> and  <code>pre_conditioner</code> are respectively, the relative tolerance, the absolute tolerance, the maximum number of iterations,  a flag indicating whether to use block variant of cg and an optional pre-conditioner.</p>"},{"location":"influence/influence_function_model/#linear-time-stochastic-second-order-approximation-lissa","title":"Linear time Stochastic Second-Order Approximation (LiSSA)","text":"<p>The LiSSA method is a stochastic approximation of the inverse Hessian vector product. Compared to conjugate gradient it is faster but less accurate and typically suffers from instability.</p> <p>In order to find the solution of the HVP, LiSSA iteratively approximates the inverse of the Hessian matrix with the following update:</p> \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\] <p>where \\(d\\) and \\(s\\) are a dampening and a scaling factor, which are essential for the convergence of the method and they need to be chosen carefully, and I is the identity matrix. More info on the theory of LiSSA can be found in the original paper (Agarwal et al., 2017)<sup>5</sup>.</p> <pre><code>from pydvl.influence.torch import LissaInfluence\nif_model = LissaInfluence(\n   model,\n   loss,\n   hessian_regularization=0.0 \n   maxiter=1000,\n   dampen=0.0,\n   scale=10.0,\n   h0=None,\n   rtol=1e-4,\n)\nif_model.fit(train_loader)\n</code></pre> <p>with the additional optional parameters <code>maxiter</code>, <code>dampen</code>, <code>scale</code>, <code>h0</code>, and <code>rtol</code>, being the maximum number of iterations, the dampening factor, the scaling factor, the initial guess for the solution and the relative tolerance, respectively.</p>"},{"location":"influence/influence_function_model/#arnoldi","title":"Arnoldi","text":"<p>The Arnoldi method is a Krylov subspace method for approximating dominating eigenvalues and eigenvectors. Under a low rank assumption on the Hessian at a minimizer (which is typically observed for deep neural networks), this approximation captures the essential action of the Hessian. More concretely, for \\(Hx=b\\) the solution is approximated by</p> \\[x \\approx V D^{-1} V^T b\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors. See also  (Schioppa et al., 2022)<sup>6</sup>.</p> <pre><code>from pydvl.influence.torch import ArnoldiInfluence\nif_model = ArnoldiInfluence(\n    model,\n    loss,\n    hessian_regularization=0.0,\n    rank_estimate=10,\n    tol=1e-6,\n)\nif_model.fit(train_loader)\n</code></pre>"},{"location":"influence/influence_function_model/#eigenvalue-corrected-k-fac","title":"Eigenvalue Corrected K-FAC","text":"<p>K-FAC, short for Kronecker-Factored Approximate Curvature, is a method that  approximates the Fisher information matrix FIM of a model.  It is possible to show that for classification models with appropriate loss  functions the FIM is equal to the Hessian of the model\u2019s loss over the dataset.  In this restricted but nonetheless important context K-FAC offers an efficient  way to approximate the Hessian and hence the influence scores.  For more info and details refer to the original paper (Martens and Grosse, 2015)<sup>7</sup>.</p> <p>The K-FAC method is implemented in the class EkfacInfluence .  The following code snippet shows how to use the K-FAC method to calculate the  influence function of a model. Note that, in contrast to the other methods for  influence function calculation, K-FAC does not require the loss function as an  input. This is because the current implementation is only applicable to  classification models with a cross entropy loss function. </p> <p><pre><code>from pydvl.influence.torch import EkfacInfluence\nif_model = EkfacInfluence(\n    model,\n    hessian_regularization=0.0,\n)\nif_model.fit(train_loader)\n</code></pre> Upon initialization, the K-FAC method will parse the model and extract which  layers require grad and which do not. Then it will only calculate the influence  scores for the layers that require grad. The current implementation of the  K-FAC method is only available for linear layers, and therefore if the model  contains non-linear layers that require gradient the K-FAC method will raise a  NotImplementedLayerRepresentationException.</p> <p>A further improvement of the K-FAC method is the Eigenvalue Corrected  K-FAC (EKFAC) method (George et al., 2018)<sup>8</sup>, which allows to further re-fit the  eigenvalues of the Hessian, thus providing a more accurate approximation.  On top of the K-FAC method, the EKFAC method is implemented by setting  <code>update_diagonal=True</code> when initialising EkfacInfluence .  The following code snippet shows how to use the EKFAC method to calculate the  influence function of a model. </p> <pre><code>from pydvl.influence.torch import EkfacInfluence\nif_model = EkfacInfluence(\n    model,\n    update_diagonal=True,\n    hessian_regularization=0.0,\n)\nif_model.fit(train_loader)\n</code></pre>"},{"location":"influence/influence_function_model/#nystrom-sketch-and-solve","title":"Nystr\u00f6m Sketch-and-Solve","text":"<p>This approximation is based on a Nystr\u00f6m low-rank approximation of the form</p> \\[\\begin{align*} H_{\\text{nys}} &amp;= (H\\Omega)(\\Omega^TH\\Omega)^{\\dagger}(H\\Omega)^T \\\\\\ &amp;= U \\Lambda U^T, \\end{align*}\\] <p>where \\((\\cdot)^{\\dagger}\\) denotes the Moore-Penrose inverse, in combination with the Sherman\u2013Morrison\u2013Woodbury formula to calculate the action of its inverse:</p> \\[\\begin{equation*}  (H_{\\text{nys}} + \\lambda I)^{-1}x = U(\\Lambda+\\lambda I)U^Tx + \\frac{1}{\\lambda}(I\u2212UU^T)x, \\end{equation*}\\] <p>see also (Hataya and Yamada, 2023)<sup>9</sup> and (Frangella et al., 2023)<sup>4</sup>. The essential  parameter is the rank of the approximation.</p> <pre><code>from pydvl.influence.torch import NystroemSketchInfluence\nif_model = NystroemSketchInfluence(\n    model,\n    loss,\n    rank=10,\n    hessian_regularization=0.0,\n)\nif_model.fit(train_loader)\n</code></pre> <p>These implementations represent the calculation logic on in memory tensors.  To scale up to large collection of data, we map these influence function models  over these collections. For a detailed discussion see the documentation page Scaling Computation.</p> <ol> <li> <p>Trefethen, L.N., Bau, D., Iii, 1997. Numerical Linear Algebra. Society for Industrial and Applied Mathematics. https://doi.org/10.1137/1.9780898719574 \u21a9</p> </li> <li> <p>Ji, H., Li, Y., 2017. A breakdown-free block conjugate gradient method. Bit Numer Math 57, 379\u2013403. https://doi.org/10.1007/s10543-016-0631-z \u21a9</p> </li> <li> <p>Bekas, C., Kokiopoulou, E., Saad, Y., 2007. An estimator for the diagonal of a matrix. Applied Numerical Mathematics, Numerical Algorithms, Parallelism and Applications (2) 57, 1214\u20131229. https://doi.org/10.1016/j.apnum.2007.01.003 \u21a9</p> </li> <li> <p>Frangella, Z., Tropp, J.A., Udell, M., 2023. Randomized Nystr\u00f6m Preconditioning. SIAM J. Matrix Anal. Appl. 44, 718\u2013752. https://doi.org/10.1137/21M1466244 \u21a9\u21a9</p> </li> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. JMLR 18, 1\u201340.\u00a0\u21a9</p> </li> <li> <p>Schioppa, A., Zablotskaia, P., Vilar, D., Sokolov, A., 2022. Scaling Up Influence Functions. Proc. AAAI Conf. Artif. Intell. 36, 8179\u20138186. https://doi.org/10.1609/aaai.v36i8.20791 \u21a9</p> </li> <li> <p>Martens, J., Grosse, R., 2015. Optimizing Neural Networks with Kronecker-factored Approximate Curvature, in: Proceedings of the 32nd International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 2408\u20132417.\u00a0\u21a9</p> </li> <li> <p>George, T., Laurent, C., Bouthillier, X., Ballas, N., Vincent, P., 2018. Fast Approximate Natural Gradient Descent in a Kronecker Factored Eigenbasis, in: Advances in Neural Information Processing Systems. Curran Associates, Inc.\u00a0\u21a9</p> </li> <li> <p>Hataya, R., Yamada, M., 2023. Nystr\u00f6m Method for Accurate and Scalable Implicit Differentiation, in: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 4643\u20134654.\u00a0\u21a9</p> </li> </ol>"},{"location":"influence/scaling_computation/","title":"Scaling Influence Computation","text":"<p>The implementations of InfluenceFunctionModel provide a convenient way to calculate influences for in memory tensors. </p> <p>Nevertheless, there is a need for computing the influences on batches of data. This might happen, if your input data does not fit into memory (e.g. it is very high-dimensional) or for large models the derivative computations exceed your memory or any combinations of these. For this scenario, we want to map our influence function model over collections of batches (or chunks) of data.</p>"},{"location":"influence/scaling_computation/#sequential","title":"Sequential","text":"<p>The simplest way is to use a double for-loop to iterate over the batches sequentially and collect them. pyDVL provides the simple convenience class SequentialInfluenceCalculator to do this. The batch size should be chosen as large as possible, such that the corresponding batches fit into memory.</p> <p><pre><code>from pydvl.influence import SequentialInfluenceCalculator\nfrom pydvl.influence.torch.util import (\n    NestedTorchCatAggregator, \n    TorchNumpyConverter,\n)\nfrom pydvl.influence.torch import CgInfluence\n\nbatch_size = 10\ntrain_dataloader = DataLoader(..., batch_size=batch_size)\ntest_dataloader = DataLoader(..., batch_size=batch_size)\n\ninfl_model = CgInfluence(model, loss, hessian_regularization=0.01)\ninfl_model = infl_model.fit(train_dataloader)\n\ninfl_calc = SequentialInfluenceCalculator(infl_model)\n\n# this does not trigger the computation\nlazy_influences = infl_calc.influences(test_dataloader, train_dataloader)\n\n# trigger computation and pull the result into main memory, \n# result is the full tensor for all combinations of the two loaders\ninfluences = lazy_influences.compute(aggregator=NestedTorchCatAggregator())\n# or\n# trigger computation and write results chunk-wise to disk using zarr \n# in a sequential manner\nlazy_influences.to_zarr(\"local_path/or/url\", TorchNumpyConverter())\n</code></pre> When invoking the <code>compute</code> method, you have the option to specify a custom aggregator  by implementing NestedSequenceAggregator.  This allows for the aggregation of computed chunks.  Such an approach is particularly beneficial for straightforward aggregation tasks,  commonly seen in sequential computation models.  Examples include operations like concatenation, as implemented in  NestedTorchCatAggregator,  or basic min and max operations. </p> <p>For more intricate aggregations, such as an argmax operation,  it's advisable to use the  DaskInfluenceCalculator  (refer to Parallel for more details). This is because it returns data structures in the  form of dask.array.Array objects, which offer an API almost fully  compatible with NumPy arrays.</p>"},{"location":"influence/scaling_computation/#parallel","title":"Parallel","text":"<p>While the sequential calculation helps in the case the resulting tensors are too large to fit into memory,  the batches are computed one after another. Because the influence computation itself is completely data parallel, you may want to use a parallel processing framework. </p> <p>pyDVL provides an implementation of a parallel computation model using dask. The wrapper class DaskInfluenceCalculator has convenience methods to map the influence function computation over chunks of data in a parallel manner.</p> <p>Again, choosing an appropriate chunk size can be crucial. For a better understanding see the official  dask best practice documentation and the following blog entry.</p> <p>Warning</p> <p>Make sure to set <code>threads_per_worker=1</code>, when using the distributed scheduler for computing, if your implementation of InfluenceFunctionModel is not thread-safe. <pre><code>client = Client(threads_per_worker=1)\n</code></pre> For details on dask schedulers see the official documentation.</p> <p><pre><code>import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom pydvl.influence import DaskInfluenceCalculator\nfrom pydvl.influence.torch import CgInfluence\nfrom pydvl.influence.torch.util import (\n    torch_dataset_to_dask_array,\n    TorchNumpyConverter,\n)\nfrom distributed import Client\n\ntrain_data_set: Dataset = LargeDataSet(\n    ...)  # Possible some out of memory large Dataset\ntest_data_set: Dataset = LargeDataSet(\n    ...)  # Possible some out of memory large Dataset\n\ntrain_dataloader = DataLoader(train_data_set)\ninfl_model = CgInfluence(model, loss, hessian_regularization=0.01)\ninfl_model = infl_model.fit(train_dataloader)\n\n# wrap your input data into dask arrays\nchunk_size = 10\nda_x, da_y = torch_dataset_to_dask_array(train_data_set, chunk_size=chunk_size)\nda_x_test, da_y_test = torch_dataset_to_dask_array(test_data_set,\n                                                   chunk_size=chunk_size)\n\n# use only one thread for scheduling, \n# due to non-thread safety of some torch operations\nclient = Client(n_workers=4, threads_per_worker=1)\n\ninfl_calc = DaskInfluenceCalculator(infl_model, \n                                  converter=TorchNumpyConverter(\n                                      device=torch.device(\"cpu\")\n                                  ),\n                                  client=client)\nda_influences = infl_calc.influences(da_x_test, da_y_test, da_x, da_y)\n# da_influences is a dask.array.Array\n# trigger computation and write chunks to disk in parallel\nda_influences.to_zarr(\"path/or/url\")\n</code></pre> During initialization of the  DaskInfluenceCalculator,  the system verifies if all workers are operating in single-threaded mode when the provided influence_function_model is designated as not thread-safe (indicated by the <code>is_thread_safe</code> property). If this condition is not met, the initialization will raise a specific error, signaling a potential thread-safety conflict.</p> <p>To intentionally skip this safety check (e.g., for debugging purposes using the single machine synchronous scheduler), you can supply the DisableClientSingleThreadCheck type.</p> <pre><code>from pydvl.influence import DisableClientSingleThreadCheck\n\ninfl_calc = DaskInfluenceCalculator(infl_model,\n                                    TorchNumpyConverter(device=torch.device(\"cpu\")),\n                                    DisableClientSingleThreadCheck)\nda_influences = infl_calc.influences(da_x_test, da_y_test, da_x, da_y)\nda_influences.compute(scheduler=\"synchronous\")\n</code></pre>"},{"location":"value/","title":"Data valuation","text":"<p>Info</p> <p>If you want to jump right into it, skip ahead to Computing data values. If you want a quick list of applications, see Applications of data valuation. For a list of all algorithms implemented in pyDVL, see Methods.</p> <p>Data valuation is the task of assigning a number to each element of a training set which reflects its contribution to the final performance of some model trained on it. Some methods attempt to be model-agnostic, but in most cases the model is an integral part of the method. In these cases, this number is not an intrinsic property of the element of interest, but typically a function of three factors:</p> <ol> <li> <p>The dataset \\(D\\), or more generally, the distribution it was sampled from: In    some cases one only cares about values wrt. a given data set, in others    value would ideally be the (expected) contribution of a data point to any    random set \\(D\\) sampled from the same distribution. pyDVL implements methods    of the first kind.</p> </li> <li> <p>The algorithm \\(\\mathcal{A}\\) mapping the data \\(D\\) to some estimator \\(f\\) in a    model class \\(\\mathcal{F}\\). E.g. MSE minimization to find the parameters of a    linear model.</p> </li> <li> <p>The performance metric of interest \\(u\\) for the problem. When value depends on    a model, it must be measured in some way which uses it. E.g. the \\(R^2\\) score    or the negative MSE over a test set. This metric will be computed over a    held-out valuation set.</p> </li> </ol> <p>pyDVL collects algorithms for the computation of data values in this sense, mostly those derived from cooperative game theory. The methods can be found in the package [[pydvl.value]], with support from modules pydvl.utils.dataset and pydvl.utils.utility, as detailed below.</p> <p>Warning</p> <p>Be sure to read the section on the difficulties using data values.</p> <p>There are three main families of methods for data valuation: game-theoretic,  influence-based and intrinsic. As of v0.8.1 pyDVL supports the first two. Here, we focus on game-theoretic concepts and refer to the main documentation on the influence funtion for the second.</p>"},{"location":"value/#game-theoretical-methods","title":"Game theoretical methods","text":"<p>The main contenders in game-theoretic approaches are Shapley values (Ghorbani and Zou, 2019)<sup>1</sup>, (Kwon et al., 2021)<sup>2</sup>,  (Schoch et al., 2022)<sup>3</sup>, their generalization to so-called semi-values by (Kwon and Zou, 2022)<sup>4</sup> and [@wang_data_2022], and the Core (Yan and Procaccia, 2021)<sup>5</sup>. All of these are implemented in pyDVL. For a full list see Methods</p> <p>In these methods, data points are considered players in a cooperative game  whose outcome is the performance of the model when trained on subsets  (coalitions) of the data, measured on a held-out valuation set. This  outcome, or utility, must typically be computed for every subset of  the training set, so that an exact computation is \\(\\mathcal{O} (2^n)\\) in the  number of samples \\(n\\), with each iteration requiring a full re-fitting of the  model using a coalition as training set. Consequently, most methods involve  Monte Carlo approximations, and sometimes approximate utilities which are  faster to compute, e.g. proxy models (Wang et al., 2022)<sup>6</sup> or constant-cost approximations like Neural Tangent Kernels (Wu et al., 2022)<sup>7</sup>.</p> <p>The reasoning behind using game theory is that, in order to be useful, an assignment of value, dubbed valuation function, is usually required to fulfil certain requirements of consistency and \"fairness\". For instance, in some applications value should not depend on the order in which data are considered, or it should be equal for samples that contribute equally to any subset of the data (of equal size). When considering aggregated value for (sub-)sets of data there are additional desiderata, like having a value function that does not increase with repeated samples. Game-theoretic methods are all rooted in axioms that by construction ensure different desiderata, but despite their practical usefulness, none of them are either necessary or sufficient for all applications. For instance, SV methods try to equitably distribute all value among all samples, failing to identify repeated ones as unnecessary, with e.g. a zero value.</p>"},{"location":"value/#computing-data-values","title":"Computing data values","text":"<p>Using pyDVL to compute data values is a simple process that can be broken down into three steps:</p> <ol> <li>Creating a Dataset object from your data.</li> <li>Creating a Utility which ties your model to    the dataset and a scoring function.</li> <li>Computing values with a method of your choice, e.g. via    compute_shapley_values.</li> </ol>"},{"location":"value/#creating-a-dataset","title":"Creating a Dataset","text":"<p>The first item in the tuple \\((D, \\mathcal{A}, u)\\) characterising data value is the dataset. The class Dataset is a simple convenience wrapper for the train and test splits that is used throughout pyDVL. The test set will be used to evaluate a scoring function for the model.</p> <p>It can be used as follows:</p> <pre><code>import numpy as np\nfrom pydvl.utils import Dataset\nfrom sklearn.model_selection import train_test_split\nX, y = np.arange(100).reshape((50, 2)), np.arange(50)\nX_train, X_test, y_train, y_test = train_test_split(\n  X, y, test_size=0.5, random_state=16\n)\ndataset = Dataset(X_train, X_test, y_train, y_test)\n</code></pre> <p>It is also possible to construct Datasets from sklearn toy datasets for illustrative purposes using from_sklearn.</p>"},{"location":"value/#grouping-data","title":"Grouping data","text":"<p>Be it because data valuation methods are computationally very expensive, or because we are interested in the groups themselves, it can be often useful or necessary to group samples to valuate them together. GroupedDataset provides an alternative to Dataset with the same interface which allows this.</p> <p>You can see an example in action in the Spotify notebook, but here's a simple example grouping a pre-existing <code>Dataset</code>. First we construct an array mapping each index in the dataset to a group, then use from_dataset:</p> <pre><code>import numpy as np\nfrom pydvl.utils import GroupedDataset\n\n# Randomly assign elements to any one of num_groups:\ndata_groups = np.random.randint(0, num_groups, len(dataset))\ngrouped_dataset = GroupedDataset.from_dataset(dataset, data_groups)\n</code></pre>"},{"location":"value/#creating-a-utility","title":"Creating a Utility","text":"<p>In pyDVL we have slightly overloaded the name \"utility\" and use it to refer to an object that keeps track of all three items in \\((D, \\mathcal{A}, u)\\). This will be an instance of Utility which, as mentioned, is a convenient wrapper for the dataset, model and scoring function used for valuation methods.</p> <p>Here's a minimal example:</p> <pre><code>import sklearn as sk\nfrom pydvl.utils import Dataset, Utility\n\ndataset = Dataset.from_sklearn(sk.datasets.load_iris())\nmodel = sk.svm.SVC()\nutility = Utility(model, dataset)\n</code></pre> <p>The object <code>utility</code> is a callable that data valuation methods will execute with different subsets of training data. Each call will retrain the model on a subset and evaluate it on the test data using a scoring function. By default, Utility will use <code>model.score()</code>, but it is possible to use any scoring function (greater values must be better). In particular, the constructor accepts the same types as argument as sklearn.model_selection.cross_validate: a string, a scorer callable or None for the default.</p> <pre><code>utility = Utility(model, dataset, \"explained_variance\")\n</code></pre> <p><code>Utility</code> will wrap the <code>fit()</code> method of the model to cache its results. This greatly reduces computation times of Monte Carlo methods. Because of how caching is implemented, it is important not to reuse <code>Utility</code> objects for different datasets. You can read more about setting up the cache in the installation guide, and in the documentation of the caching module.</p>"},{"location":"value/#using-custom-scorers","title":"Using custom scorers","text":"<p>The <code>scoring</code> argument of Utility can be used to specify a custom Scorer object. This is a simple wrapper for a callable that takes a model, and test data and returns a score.</p> <p>More importantly, the object provides information about the range of the score, which is used by some methods by estimate the number of samples necessary, and about what default value to use when the model fails to train.</p> <p>Note</p> <p>The most important property of a <code>Scorer</code> is its default value. Because many models will fail to fit on small subsets of the data, it is important to provide a sensible default value for the score.</p> <p>It is possible to skip the construction of the Scorer when constructing the <code>Utility</code> object. The two following calls are equivalent:</p> <pre><code>from pydvl.utils import Utility, Scorer\n\nutility = Utility(\n   model, dataset, \"explained_variance\", score_range=(-np.inf, 1), default_score=0.0\n)\nutility = Utility(\n   model, dataset, Scorer(\"explained_variance\", range=(-np.inf, 1), default=0.0)\n)\n</code></pre>"},{"location":"value/#learning-the-utility","title":"Learning the utility","text":"<p>Because each evaluation of the utility entails a full retrain of the model with a new subset of the training set, it is natural to try to learn this mapping from subsets to scores. This is the idea behind Data Utility Learning (DUL)  (Wang et al., 2022)<sup>6</sup> and in pyDVL it's as simple as wrapping the <code>Utility</code> inside DataUtilityLearning:</p> <pre><code>from pydvl.utils import Utility, DataUtilityLearning, Dataset\nfrom sklearn.linear_model import LinearRegression, LogisticRegression\nfrom sklearn.datasets import load_iris\n\ndataset = Dataset.from_sklearn(load_iris())\nu = Utility(LogisticRegression(), dataset)\ntraining_budget = 3\nwrapped_u = DataUtilityLearning(u, training_budget, LinearRegression())\n\n# First 3 calls will be computed normally\nfor i in range(training_budget):\n   _ = wrapped_u((i,))\n# Subsequent calls will be computed using the fit model for DUL\nwrapped_u((1, 2, 3))\n</code></pre> <p>As you can see, all that is required is a model to learn the utility itself and the fitting and using of the learned model happens behind the scenes.</p> <p>There is a longer example with an investigation of the results achieved by DUL in a dedicated notebook.</p>"},{"location":"value/#leave-one-out-values","title":"Leave-One-Out values","text":"<p>LOO is the simplest approach to valuation. It assigns to each sample its marginal utility as value:</p> \\[v_u(i) = u(D) \u2212 u(D_{-i}).\\] <p>For notational simplicity, we consider the valuation function as defined over the indices of the dataset \\(D\\), and \\(i \\in D\\) is the index of the sample, \\(D_{-i}\\) is the training set without the sample \\(x_i\\), and \\(u\\) is the utility function. See the section on notation for more.</p> <p>For the purposes of data valuation, this is rarely useful beyond serving as a baseline for benchmarking. Although in some benchmarks it can perform astonishingly well on occasion. One particular weakness is that it does not necessarily correlate with an intrinsic value of a sample: since it is a marginal utility, it is affected by diminishing returns. Often, the training set is large enough for a single sample not to have any significant effect on training performance, despite any qualities it may possess. Whether this is indicative of low value or not depends on each one's goals and definitions, but other methods are typically preferable.</p> <pre><code>from pydvl.value.loo import compute_loo\n\nvalues = compute_loo(utility, n_jobs=-1)\n</code></pre> <p>The return value of all valuation functions is an object of type ValuationResult. This can be iterated over, indexed with integers, slices and Iterables, as well as converted to a pandas.DataFrame.</p>"},{"location":"value/#problems-of-data-values","title":"Problems of data values","text":"<p>There are a number of factors that affect how useful values can be for your project. In particular, regression can be especially tricky, but the particular nature of every (non-trivial) ML problem can have an effect:</p> <ul> <li> <p>Variance of the utility: Classical applications of game theoretic value   concepts operate with deterministic utilities, as do many of the bounds in the   literature. But in ML we use an evaluation of the model on a validation set as a   proxy for the true risk. Even if the utility is bounded, its variance will   affect final values, and even more so any Monte Carlo estimates.   Several works have tried to cope with variance. [@wang_data_2022] prove that by   relaxing one of the Shapley axioms and considering the general class of   semi-values, of which Shapley is an instance, one can prove that a choice of   constant weights is the best one can do in a utility-agnostic setting. This   method, dubbed Data Banzhaf, is available in pyDVL as   compute_banzhaf_semivalues.</p> Averaging repeated utility evaluations <p>One workaround in pyDVL is to configure the caching system to allow multiple evaluations of the utility for every index set. A moving average is  computed and returned once the standard error is small, see CachedFuncConfig. Note however that in practice, the likelihood of cache hits is low, so one would have to force recomputation manually somehow.</p> </li> <li> <p>Unbounded utility: Choosing a scorer for a classifier is simple: accuracy   or some F-score provides a bounded number with a clear interpretation. However,   in regression problems most scores, like \\(R^2\\), are not bounded because   regressors can be arbitrarily bad. This leads to great variability in the   utility for low sample sizes, and hence unreliable Monte Carlo approximations   to the values. Nevertheless, in practice it is only the ranking of samples   that matters, and this tends to be accurate (wrt. to the true ranking) despite   inaccurate values.</p> Squashing scores <p>pyDVL offers a dedicated function composition for scorer functions which can be used to squash a score. The following is defined in module score: <pre><code>import numpy as np\nfrom pydvl.utils import compose_score\n\ndef sigmoid(x: float) -&gt; float:\n  return float(1 / (1 + np.exp(-x)))\n\nsquashed_r2 = compose_score(\"r2\", sigmoid, \"squashed r2\")\n\nsquashed_variance = compose_score(\n  \"explained_variance\", sigmoid, \"squashed explained variance\"\n)\n</code></pre> These squashed scores can prove useful in regression problems, but they can also introduce issues in the low-value regime.</p> </li> <li> <p>Data set size: Computing exact Shapley values is NP-hard, and Monte Carlo   approximations can converge slowly. Massive datasets are thus impractical, at   least with game-theoretical methods. A workaround   is to group samples and investigate their value together. You can do this using   GroupedDataset. There is a fully   worked-out example here. Some algorithms   also provide different sampling strategies to reduce the variance, but due to a   no-free-lunch-type theorem, no single strategy can be optimal for all utilities.   Finally, model specific methods like   kNN-Shapley (Jia et al., 2019)<sup>8</sup>, or   altogether different and typically faster approaches like   Data-OOB (Kwon and Zou, 2023)<sup>9</sup> can also be   used. </p> </li> <li> <p>Model size: Since every evaluation of the utility entails retraining the   whole model on a subset of the data, large models require great amounts of   computation. But also, they will effortlessly interpolate small to medium   datasets, leading to great variance in the evaluation of performance on the   dedicated validation set. One mitigation for this problem is cross-validation,   but this would incur massive computational cost. As of v0.8.1 there are no   facilities in pyDVL for cross-validating the utility (note that this would   require cross-validating the whole value computation).</p> </li> </ul>"},{"location":"value/#notation-and-nomenclature","title":"Notation and nomenclature","text":"<p>Todo</p> <p>Organize this section better and use its content consistently throughout the documentation.</p> <p>The following notation is used throughout the documentation:</p> <p>Let \\(D = \\{x_1, \\ldots, x_n\\}\\) be a training set of \\(n\\) samples.</p> <p>The utility function \\(u:\\mathcal{D} \\rightarrow \\mathbb{R}\\) maps subsets of \\(D\\) to real numbers. In pyDVL, we typically call this mapping a score for consistency with sklearn, and reserve the term utility for the triple of dataset \\(D\\), model \\(f\\) and score \\(u\\), since they are used together to compute the value.</p> <p>The value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is denoted as \\(v_u(x_i)\\) or simply \\(v(i)\\).</p> <p>For any \\(S \\subseteq D\\), we denote by \\(S_{-i}\\) the set of samples in \\(D\\) excluding \\(x_i\\), and \\(S_{+i}\\) denotes the set \\(S\\) with \\(x_i\\) added.</p> <p>The marginal utility of adding sample \\(x_i\\) to a subset \\(S\\) is denoted as \\(\\delta(i) := u(S_{+i}) - u(S)\\).</p> <p>The set \\(D_{-i}^{(k)}\\) contains all subsets of \\(D\\) of size \\(k\\) that do not include sample \\(x_i\\).</p> <ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Rivas, M.A., Zou, J., 2021. Efficient Computation and Analysis of Distributional Shapley Values, in: Proceedings of the 24th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 793\u2013801.\u00a0\u21a9</p> </li> <li> <p>Schoch, S., Xu, H., Ji, Y., 2022. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification, in: Proc. Of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). Presented at the Advances in Neural Information Processing Systems (NeurIPS 2022).\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning, in: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022,. Presented at the AISTATS 2022, PMLR.\u00a0\u21a9</p> </li> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9</p> </li> <li> <p>Wang, T., Yang, Y., Jia, R., 2022. Improving Cooperative Game Theory-based Data Valuation via Data Utility Learning. Presented at the International Conference on Learning Representations (ICLR 2022). Workshop on Socially Responsible Machine Learning, arXiv. https://doi.org/10.48550/arXiv.2107.06336 \u21a9\u21a9</p> </li> <li> <p>Wu, Z., Shu, Y., Low, B.K.H., 2022. DAVINZ: Data Valuation using Deep Neural Networks at Initialization, in: Proceedings of the 39th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 24150\u201324176.\u00a0\u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Gurel, N.M., Li, B., Zhang, C., Spanos, C., Song, D., 2019. Efficient task-specific data valuation for nearest neighbor algorithms. Proc. VLDB Endow. 12, 1610\u20131623. https://doi.org/10.14778/3342263.3342637 \u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2023. Data-OOB: Out-of-bag Estimate as a Simple and Efficient Data Value, in: Proceedings of the 40th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 18135\u201318152.\u00a0\u21a9</p> </li> </ol>"},{"location":"value/classwise-shapley/","title":"Class-wise Shapley","text":"<p>Class-wise Shapley (CWS) (Schoch et al., 2022)<sup>1</sup> offers a Shapley framework tailored for classification problems.  Given a sample \\(x_i\\) with label \\(y_i \\in \\mathbb{N}\\), let \\(D_{y_i}\\) be the subset of \\(D\\) with labels \\(y_i\\), and \\(D_{-y_i}\\) be the complement of \\(D_{y_i}\\) in \\(D\\). The key idea is that the sample \\((x_i, y_i)\\) might improve the overall model performance on \\(D\\), while being detrimental for the performance on \\(D_{y_i},\\) e.g. because of a wrong label. To address this issue, the authors introduced</p> \\[ v_u(i) = \\frac{1}{2^{|D_{-y_i}|}} \\sum_{S_{-y_i}} \\left [ \\frac{1}{|D_{y_i}|}\\sum_{S_{y_i}} \\binom{|D_{y_i}|-1}{|S_{y_i}|}^{-1} \\delta(S_{y_i} | S_{-y_i}) \\right ], \\] <p>where \\(S_{y_i} \\subseteq D_{y_i} \\setminus \\{i\\}\\) and \\(S_{-y_i} \\subseteq D_{-y_i}\\) is arbitrary (in particular, not the complement of \\(S_{y_i}\\)). The function \\(\\delta\\) is called set-conditional marginal Shapley value and is defined as</p> \\[ \\delta(S | C) = u( S_{+i} | C ) \u2212 u(S | C), \\] <p>for any set \\(S\\) such that \\(i \\notin S, C\\) and \\(S \\cap C = \\emptyset\\).</p> <p>In practical applications, estimating this quantity is done both with Monte Carlo sampling of the powerset, and the set of index permutations  (Castro et al., 2009)<sup>2</sup>. Typically, this requires fewer samples than the original Shapley value, although the actual speed-up depends on the model and the dataset.</p> <p>Computing classwise Shapley values</p> <p>Like all other game-theoretic valuation methods, CWS requires a Utility object constructed with model and dataset, with the peculiarity of requiring a specific ClasswiseScorer. The entry point is the function compute_classwise_shapley_values:</p> <pre><code>from pydvl.value import *\n\nmodel = ...\ndata = Dataset(...)\nscorer = ClasswiseScorer(...)\nutility = Utility(model, data, scorer)\nvalues = compute_classwise_shapley_values(\n    utility,\n    done=HistoryDeviation(n_steps=500, rtol=5e-2) | MaxUpdates(5000),\n    truncation=RelativeTruncation(utility, rtol=0.01),\n    done_sample_complements=MaxChecks(1),\n    normalize_values=True\n)\n</code></pre>"},{"location":"value/classwise-shapley/#the-class-wise-scorer","title":"The class-wise scorer","text":"<p>In order to use the classwise Shapley value, one needs to define a ClasswiseScorer. This scorer is defined as</p> \\[ u(S) = f(a_S(D_{y_i})) g(a_S(D_{-y_i})), \\] <p>where \\(f\\) and \\(g\\) are monotonically increasing functions, \\(a_S(D_{y_i})\\) is the in-class accuracy, and \\(a_S(D_{-y_i})\\) is the out-of-class accuracy (the names originate from a choice by the authors to use accuracy, but in principle any other score, like \\(F_1\\) can be used). </p> <p>The authors show that \\(f(x)=x\\) and \\(g(x)=e^x\\) have favorable properties and are therefore the defaults, but we leave the option to set different functions \\(f\\) and \\(g\\) for an exploration with different base scores. </p> <p>The default class-wise scorer</p> <p>Constructing the CWS scorer requires choosing a metric and the functions \\(f\\) and \\(g\\):</p> <pre><code>import numpy as np\nfrom pydvl.value.shapley.classwise import ClasswiseScorer\n\n# These are the defaults\nidentity = lambda x: x\nscorer = ClasswiseScorer(\n    \"accuracy\",\n    in_class_discount_fn=identity,\n    out_of_class_discount_fn=np.exp\n)\n</code></pre> Surface of the discounted utility function <p>The level curves for \\(f(x)=x\\) and \\(g(x)=e^x\\) are depicted below. The lines illustrate the contour lines, annotated with their respective gradients. Level curves of the class-wise utility </p>"},{"location":"value/classwise-shapley/#evaluation","title":"Evaluation","text":"<p>We illustrate the method with two experiments: point removal and noise removal, as well as an analysis of the distribution of the values. For this we employ the nine datasets used in (Schoch et al., 2022)<sup>1</sup>, using the same pre-processing. For images, PCA is used to reduce down to 32 the features found by a pre-trained <code>Resnet18</code> model. Standard loc-scale normalization is performed for all models except gradient boosting, since the latter is not sensitive to the scale of the features.</p> Datasets used for evaluation Dataset Data Type Classes Input Dims OpenML ID Diabetes Tabular 2 8 37 Click Tabular 2 11 1216 CPU Tabular 2 21 197 Covertype Tabular 7 54 1596 Phoneme Tabular 2 5 1489 FMNIST Image 2 32 40996 CIFAR10 Image 2 32 40927 MNIST (binary) Image 2 32 554 MNIST (multi) Image 10 32 554 <p>We show mean and coefficient of variation (CV) \\(\\frac{\\sigma}{\\mu}\\) of an \"inner metric\". The former shows the performance of the method, whereas the latter displays its stability: we normalize by the mean to see the relative effect of the standard deviation. Ideally the mean value is maximal and CV minimal. </p> <p>Finally, we note that for all sampling-based valuation methods the same number of evaluations of the marginal utility was used. This is important to make the algorithms comparable, but in practice one should consider using a more sophisticated stopping criterion.</p>"},{"location":"value/classwise-shapley/#dataset-pruning-for-logistic-regression-point-removal","title":"Dataset pruning for logistic regression (point removal)","text":"<p>In (best-)point removal, one first computes values for the training set and then removes in sequence the points with the highest values. After each removal, the remaining points are used to train the model from scratch and performance is measured on a test set. This produces a curve of performance vs. number of points removed which we show below.</p> <p>As a scalar summary of this curve, (Schoch et al., 2022)<sup>1</sup> define Weighted Accuracy Drop (WAD) as:</p> \\[ \\text{WAD} =  \\sum_{j=1}^{n} \\left ( \\frac{1}{j} \\sum_{i=1}^{j}  a_{T_{-\\{1 \\colon i-1 \\}}}(D) - a_{T_{-\\{1 \\colon i \\}}}(D) \\right) = a_T(D) - \\sum_{j=1}^{n} \\frac{a_{T_{-\\{1 \\colon j \\}}}(D)}{j} , \\] <p>where \\(a_T(D)\\) is the accuracy of the model (trained on \\(T\\)) evaluated on \\(D\\) and \\(T_{-\\{1 \\colon j \\}}\\) is the set \\(T\\) without elements from \\(\\{1, \\dots , j \\}\\).</p> <p>We run the point removal experiment for a logistic regression model five times and compute WAD for each run, then report the mean \\(\\mu_\\text{WAD}\\) and standard deviation \\(\\sigma_\\text{WAD}\\).</p> <p> Mean WAD for best-point removal on logistic regression. Values computed using LOO, CWS, Beta Shapley, and TMCS </p> <p>We see that CWS is competitive with all three other methods. In all problems except <code>MNIST (multi)</code> it outperforms TMCS, while in that case TMCS has a slight advantage.</p> <p>In order to understand the variability of WAD we look at its coefficient of variation (lower is better):</p> <p> Coefficient of Variation of WAD for best-point removal on logistic regression. Values computed using LOO, CWS, Beta Shapley, and TMCS </p> <p>CWS is not the best method in terms of CV. For <code>CIFAR10</code>, <code>Click</code>, <code>CPU</code> and <code>MNIST (binary)</code> Beta Shapley has the lowest CV. For <code>Diabetes</code>, <code>MNIST (multi)</code> and <code>Phoneme</code> CWS is the winner and for <code>FMNIST</code> and <code>Covertype</code> TMCS takes the lead. Besides LOO, TMCS has the highest relative standard deviation.</p> <p>The following plot shows accuracy vs number of samples removed. Random values serve as a baseline. The shaded area represents the 95% bootstrap confidence interval of the mean across 5 runs.</p> <p> Accuracy after best-sample removal using values from logistic  regression </p> <p>Because samples are removed from high to low valuation order, we expect a steep decrease in the curve.</p> <p>Overall we conclude that in terms of mean WAD, CWS and TMCS perform best, with CWS's CV on par with Beta Shapley's, making CWS a competitive method.</p>"},{"location":"value/classwise-shapley/#dataset-pruning-for-a-neural-network-by-value-transfer","title":"Dataset pruning for a neural network by value transfer","text":"<p>Transfer of values from one model to another is probably of greater practical relevance: values are computed using a cheap model and used to prune the dataset before training a more expensive one.</p> <p>The following plot shows accuracy vs number of samples removed for transfer from logistic regression to a neural network. The shaded area represents the 95% bootstrap confidence interval of the mean across 5 runs.</p> <p> Accuracy after sample removal using values transferred from logistic regression to an MLP </p> <p>As in the previous experiment samples are removed from high to low valuation order and hence we expect a steep decrease in the curve. CWS is competitive with the other methods, especially in very unbalanced datasets like <code>Click</code>. In other datasets, like <code>Covertype</code>, <code>Diabetes</code> and <code>MNIST (multi)</code> the performance is on par with TMCS.</p>"},{"location":"value/classwise-shapley/#detection-of-mis-labeled-data-points","title":"Detection of mis-labeled data points","text":"<p>The next experiment tries to detect mis-labeled data points in binary classification tasks. 20% of the indices is flipped at random (we don't consider multi-class datasets because there isn't a unique flipping strategy). The following table shows the mean of the area under the curve (AUC) for five runs.</p> <p> Mean AUC for mis-labeled data point detection. Values computed using LOO, CWS, Beta Shapley, and  TMCS </p> <p>In the majority of cases TMCS has a slight advantage over CWS, except for <code>Click</code>, where CWS has a slight edge, most probably due to the unbalanced nature of the dataset. The following plot shows the CV for the AUC of the five runs.</p> <p> Coefficient of variation of AUC for mis-labeled data point detection. Values computed using LOO, CWS, Beta Shapley, and TMCS </p> <p>In terms of CV, CWS has a clear edge over TMCS and Beta Shapley.</p> <p>Finally, we look at the ROC curves training the classifier on the \\(n\\) first samples in increasing order of valuation (i.e. starting with the worst):</p> <p> Mean ROC across 5 runs with 95% bootstrap CI </p> <p>Although at first sight TMCS seems to be the winner, CWS stays competitive after factoring in running time. For a perfectly balanced dataset, CWS needs on average fewer samples than TCMS.</p>"},{"location":"value/classwise-shapley/#value-distribution","title":"Value distribution","text":"<p>For illustration, we compare the distribution of values computed by TMCS and CWS.</p> <p> Histogram and estimated density of the values computed by TMCS and CWS on all nine datasets </p> <p>For <code>Click</code> TMCS has a multi-modal distribution of values. We hypothesize that this is due to the highly unbalanced nature of the dataset, and notice that CWS has a single mode, leading to its greater performance on this dataset.</p>"},{"location":"value/classwise-shapley/#conclusion","title":"Conclusion","text":"<p>CWS is an effective way to handle classification problems, in particular for unbalanced datasets. It reduces the computing requirements by considering in-class and out-of-class points separately.</p> <ol> <li> <p>Schoch, S., Xu, H., Ji, Y., 2022. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification, in: Proc. Of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). Presented at the Advances in Neural Information Processing Systems (NeurIPS 2022).\u00a0\u21a9\u21a9\u21a9</p> </li> <li> <p>Castro, J., G\u00f3mez, D., Tejada, J., 2009. Polynomial calculation of the Shapley value based on sampling. Computers   &amp; Operations Research, Selected papers presented at the Tenth International Symposium on Locational Decisions (ISOLDE X) 36, 1726\u20131730. https://doi.org/10.1016/j.cor.2008.04.004 \u21a9</p> </li> </ol>"},{"location":"value/semi-values/","title":"Semi-values","text":"<p>SV is a particular case of a more general concept called semi-value, which is a generalization to different weighting schemes. A semi-value is any valuation function with the form:</p> \\[ v_\\text{semi}(i) = \\sum_{i=1}^n w(k) \\sum_{S \\subset D_{-i}^{(k)}} [u(S_{+i}) - u(S)], \\] <p>where the coefficients \\(w(k)\\) satisfy the property:</p> \\[\\sum_{k=1}^n w(k) = 1,\\] <p>the set \\(D_{-i}^{(k)}\\) contains all subsets of \\(D\\) of size \\(k\\) that do not include sample \\(x_i\\), \\(S_{+i}\\) is the set \\(S\\) with \\(x_i\\) added, and \\(u\\) is the utility function.</p> <p>Two instances of this are Banzhaf indices (Wang and Jia, 2023)<sup>1</sup>, and Beta Shapley (Kwon and Zou, 2022)<sup>2</sup>, with better numerical and rank stability in certain situations.</p> <p>Note</p> <p>Shapley values are a particular case of semi-values and can therefore also be computed with the methods described here. However, as of version 0.8.1, we recommend using compute_shapley_values instead, in particular because it implements truncation policies for TMCS.</p>"},{"location":"value/semi-values/#beta-shapley","title":"Beta Shapley","text":"<p>For some machine learning applications, where the utility is typically the performance when trained on a set \\(S \\subset D\\), diminishing returns are often observed when computing the marginal utility of adding a new data point.</p> <p>Beta Shapley is a weighting scheme that uses the Beta function to place more weight on subsets deemed to be more informative. The weights are defined as:</p> \\[ w(k) := \\frac{B(k+\\beta, n-k+1+\\alpha)}{B(\\alpha, \\beta)}, \\] <p>where \\(B\\) is the Beta function, and \\(\\alpha\\) and \\(\\beta\\) are parameters that control the weighting of the subsets. Setting both to 1 recovers Shapley values, and setting \\(\\alpha = 1\\), and \\(\\beta = 16\\) is reported in (Kwon and Zou, 2022)<sup>2</sup> to be a good choice for some applications. Beta Shapley values are available in pyDVL through compute_beta_shapley_semivalues:</p> <pre><code>from pydvl.value import *\n\nutility = Utility(model, data)\nvalues = compute_beta_shapley_semivalues(\n    u=utility, done=AbsoluteStandardError(threshold=1e-4), alpha=1, beta=16\n)\n</code></pre> <p>See however the Banzhaf indices section  for an alternative choice of weights which is reported to work better.</p>"},{"location":"value/semi-values/#banzhaf-indices","title":"Banzhaf indices","text":"<p>As noted in the section Problems of Data Values, the Shapley value can be very sensitive to variance in the utility function. For machine learning applications, where the utility is typically the performance when trained on a set \\(S \\subset D\\), this variance is often largest for smaller subsets \\(S\\). It is therefore reasonable to try reducing the relative contribution of these subsets with adequate weights.</p> <p>One such choice of weights is the Banzhaf index, which is defined as the constant:</p> \\[w(k) := 2^{n-1},\\] <p>for all set sizes \\(k\\). The intuition for picking a constant weight is that for any choice of weight function \\(w\\), one can always construct a utility with higher variance where \\(w\\) is greater. Therefore, in a worst-case sense, the best one can do is to pick a constant weight.</p> <p>The authors of (Wang and Jia, 2023)<sup>1</sup> show that Banzhaf indices are more robust to variance in the utility function than Shapley and Beta Shapley values. They are available in pyDVL through compute_banzhaf_semivalues:</p> <pre><code>from pydvl.value import *\n\nutility = Utility(model, data)\nvalues = compute_banzhaf_semivalues(\n    u=utility, done=AbsoluteStandardError(threshold=1e-4), alpha=1, beta=16\n)\n</code></pre>"},{"location":"value/semi-values/#banzhaf-semi-values-with-msr-sampling","title":"Banzhaf semi-values with MSR sampling","text":"<p>Wang et. al. propose a more sample-efficient method for computing Banzhaf  semivalues in their paper Data Banzhaf: A Robust Data Valuation Framework  for Machine Learning (Wang and Jia, 2023)<sup>1</sup>. This method updates all semivalues per evaluation of the utility (i.e. per model trained) based on whether a  specific data point was included in the data subset or not. The expression  for computing the semivalues is</p> \\[\\hat{\\phi}_{MSR}(i) = \\frac{1}{|\\mathbf{S}_{\\ni i}|} \\sum_{S \\in  \\mathbf{S}_{\\ni i}} U(S) - \\frac{1}{|\\mathbf{S}_{\\not{\\ni} i}|}  \\sum_{S \\in \\mathbf{S}_{\\not{\\ni} i}} U(S)\\] <p>where \\(\\mathbf{S}_{\\ni i}\\) are the subsets that contain the index \\(i\\) and  \\(\\mathbf{S}_{\\not{\\ni} i}\\) are the subsets not containing the index \\(i\\).</p> <p>The function implementing this method is compute_msr_banzhaf_semivalues.</p> <p><pre><code>from pydvl.value import compute_msr_banzhaf_semivalues, RankCorrelation, Utility\n\nutility = Utility(model, data)\nvalues = compute_msr_banzhaf_semivalues(\n  u=utility, done=RankCorrelation(rtol=0.001),\n  )\n</code></pre> For further details on how to use this method and a comparison of the sample  efficiency, we suggest to take a look at the example notebook  msr_banzhaf_spotify.</p>"},{"location":"value/semi-values/#general-semi-values","title":"General semi-values","text":"<p>As explained above, both Beta Shapley and Banzhaf indices are special cases of semi-values. In pyDVL we provide a general method for computing these with any combination of the three ingredients that define a semi-value:</p> <ul> <li>A utility function \\(u\\).</li> <li>A sampling method</li> <li>A weighting scheme \\(w\\).</li> </ul> <p>You can construct any combination of these three ingredients with compute_generic_semivalues. The utility function is the same as for Shapley values, and the sampling method can be any of the types defined in the samplers module. For instance, the following snippet is equivalent to the above:</p> <pre><code>from pydvl.value import *\n\ndata = Dataset(...)\nutility = Utility(model, data)\nvalues = compute_generic_semivalues(\n  sampler=PermutationSampler(data.indices),\n  u=utility,\n  coefficient=beta_coefficient(alpha=1, beta=16),\n  done=AbsoluteStandardError(threshold=1e-4),\n)\n</code></pre> <p>Allowing any coefficient can help when experimenting with models which are more sensitive to changes in training set size. However, Data Banzhaf indices are proven to be the most robust to variance in the utility function, in the sense of rank stability, across a range of models and datasets (Wang and Jia, 2023)<sup>1</sup>. </p> <p>Careful with permutation sampling</p> <p>This generic implementation of semi-values allowing for any combination of sampling and weighting schemes is very flexible and, in principle, it recovers the original Shapley value, so that  compute_shapley_values is no longer necessary. However, it loses the optimization in permutation sampling that reuses the utility computation from the last iteration when iterating over a permutation. This doubles the computation requirements (and slightly increases variance) when using permutation sampling, unless the cache is enabled. In addition, as mentioned above, truncation policies are not supported by this generic implementation (as of v0.8.1). For these reasons it is preferable to use compute_shapley_values whenever not computing other semi-values.</p> <ol> <li> <p>Wang, J.T., Jia, R., 2023. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning, in: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 6388\u20136421.\u00a0\u21a9\u21a9\u21a9\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning, in: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022,. Presented at the AISTATS 2022, PMLR.\u00a0\u21a9\u21a9</p> </li> </ol>"},{"location":"value/shapley/","title":"Shapley value","text":""},{"location":"value/shapley/#shapley-value","title":"Shapley value","text":"<p>The Shapley method is an approach to compute data values originating in cooperative game theory. Shapley values are a common way of assigning payoffs to each participant in a cooperative game (i.e. one in which players can form coalitions) in a way that ensures that certain axioms are fulfilled.</p> <p>pyDVL implements several methods for the computation and approximation of Shapley values. They can all be accessed via the facade function compute_shapley_values. The supported methods are enumerated in ShapleyMode.</p> <p>Empirically, the most useful method is the so-called Truncated Monte Carlo Shapley (Ghorbani and Zou, 2019)<sup>1</sup>, which is a Monte Carlo approximation of the permutation Shapley value.</p>"},{"location":"value/shapley/#combinatorial-shapley","title":"Combinatorial Shapley","text":"<p>The first algorithm is just a verbatim implementation of the definition. As such it returns as exact a value as the utility function allows (see what this means in Problems of Data Values).</p> <p>The value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is computed as a weighted sum of its marginal utility wrt. every possible coalition of training samples within the training set:</p> \\[ v(i) = \\frac{1}{n} \\sum_{S \\subseteq D_{-i}} \\binom{n-1}{ | S | }^{-1} [u(S_{+i}) \u2212 u(S)] ,\\] <p>where \\(D_{-i}\\) denotes the set of samples in \\(D\\) excluding \\(x_i\\), and \\(S_{+i}\\) denotes the set \\(S\\) with \\(x_i\\) added.</p> <pre><code>from pydvl.value import compute_shapley_values\n\nvalues = compute_shapley_values(utility, mode=\"combinatorial_exact\")\ndf = values.to_dataframe(column='value')\n</code></pre> <p>We can convert the return value to a pandas.DataFrame. and name the column with the results as <code>value</code>. Please refer to the documentation in shapley and ValuationResult for more information.</p>"},{"location":"value/shapley/#monte-carlo-combinatorial-shapley","title":"Monte Carlo Combinatorial Shapley","text":"<p>Because the number of subsets \\(S \\subseteq D_{-i}\\) is \\(2^{ | D | - 1 }\\), one typically must resort to approximations. The simplest one is done via Monte Carlo sampling of the powerset \\(\\mathcal{P}(D)\\). In pyDVL this simple technique is called \"Monte Carlo Combinatorial\". The method has very poor converge rate and others are preferred, but if desired, usage follows the same pattern:</p> <pre><code>from pydvl.value import compute_shapley_values, MaxUpdates\n\nvalues = compute_shapley_values(\n   utility, mode=\"combinatorial_montecarlo\", done=MaxUpdates(1000)\n)\ndf = values.to_dataframe(column='cmc')\n</code></pre> <p>The DataFrames returned by most Monte Carlo methods will contain approximate standard errors as an additional column, in this case named <code>cmc_stderr</code>.</p> <p>Note the usage of the object MaxUpdates as the stop condition. This is an instance of a StoppingCriterion. Other examples are MaxTime and AbsoluteStandardError.</p>"},{"location":"value/shapley/#owen-sampling","title":"Owen sampling","text":"<p>Owen Sampling (Okhrati and Lipani, 2021)<sup>2</sup> is a practical algorithm based on the combinatorial definition. It uses a continuous extension of the utility from \\(\\{0,1\\}^n\\), where a 1 in position \\(i\\) means that sample \\(x_i\\) is used to train the model, to \\([0,1]^n\\). The ensuing expression for Shapley value uses integration instead of discrete weights:</p> \\[ v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{-i})} [u(S_{+i}) - u(S)]. \\] <p>Using Owen sampling follows the same pattern as every other method for Shapley values in pyDVL. First construct the dataset and utility, then call compute_shapley_values:</p> <pre><code>from pydvl.value import compute_shapley_values\n\nvalues = compute_shapley_values(\n   u=utility, mode=\"owen\", n_iterations=4, max_q=200\n)\n</code></pre> <p>There are more details on Owen sampling, and its variant Antithetic Owen Sampling in the documentation for the function doing the work behind the scenes: owen_sampling_shapley.</p> <p>Note that in this case we do not pass a StoppingCriterion to the function, but instead the number of iterations and the maximum number of samples to use in the integration.</p>"},{"location":"value/shapley/#permutation-shapley","title":"Permutation Shapley","text":"<p>An equivalent way of computing Shapley values (<code>ApproShapley</code>) appeared in  (Castro et al., 2009)<sup>3</sup> and is the basis for the method most often used in practice. It uses permutations over indices instead of subsets:</p> \\[ v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{:i} \\cup \\{x_i\\}) \u2212 u(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears. To approximate this sum (which has \\(\\mathcal{O}(n!)\\) terms!) one uses Monte Carlo sampling of permutations, something which has surprisingly low sample complexity. One notable difference wrt. the combinatorial approach above is that the approximations always fulfill the efficiency axiom of Shapley, namely \\(\\sum_{i=1}^n \\hat{v}_i = u(D)\\) (see  (Castro et al., 2009)<sup>3</sup>, Proposition 3.2).</p> <p>By adding two types of early stopping, the result is the so-called Truncated Monte Carlo Shapley (Ghorbani and Zou, 2019)<sup>1</sup>, which is efficient enough to be useful in applications. The first is simply a convergence criterion, of which there are several to choose from. The second is a criterion to truncate the iteration over single permutations. RelativeTruncation chooses to stop iterating over samples in a permutation when the marginal utility becomes too small.</p> <pre><code>from pydvl.value import compute_shapley_values, MaxUpdates, RelativeTruncation\n\nvalues = compute_shapley_values(\n    u=utility,\n    mode=\"permutation_montecarlo\",\n    done=MaxUpdates(1000),\n    truncation=RelativeTruncation(utility, rtol=0.01)\n)\n</code></pre> <p>You can see this method in action in this example using the Spotify dataset.</p>"},{"location":"value/shapley/#exact-shapley-for-knn","title":"Exact Shapley for KNN","text":"<p>It is possible to exploit the local structure of K-Nearest Neighbours to reduce the amount of subsets to consider: because no sample besides the K closest affects the score, most are irrelevant and it is possible to compute a value in linear time. This method was introduced by (Jia et al., 2019)<sup>4</sup>, and can be used in pyDVL with:</p> <pre><code>from pydvl.utils import Dataset, Utility\nfrom pydvl.value import compute_shapley_values\nfrom sklearn.neighbors import KNeighborsClassifier\n\nmodel = KNeighborsClassifier(n_neighbors=5)\ndata = Dataset(...)\nutility = Utility(model, data)\nvalues = compute_shapley_values(u=utility, mode=\"knn\")\n</code></pre>"},{"location":"value/shapley/#group-testing","title":"Group testing","text":"<p>An alternative method for the approximation of Shapley values introduced in  (Jia et al., 2019)<sup>4</sup> first estimates the differences of values with a Monte Carlo sum. With</p> \\[\\hat{\\Delta}_{i j} \\approx v_i - v_j,\\] <p>one then solves the following linear constraint satisfaction problem (CSP) to infer the final values:</p> \\[ \\begin{array}{lll} \\sum_{i = 1}^N v_i &amp; = &amp; U (D)\\\\ | v_i - v_j - \\hat{\\Delta}_{i j} | &amp; \\leqslant &amp; \\frac{\\varepsilon}{2 \\sqrt{N}} \\end{array} \\] <p>Warning</p> <p>We have reproduced this method in pyDVL for completeness and benchmarking, but we don't advocate its use because of the speed and memory cost. Despite our best efforts, the number of samples required in practice for convergence can be several orders of magnitude worse than with e.g. TMCS. Additionally, the CSP can sometimes turn out to be infeasible.</p> <p>Usage follows the same pattern as every other Shapley method, but with the addition of an <code>epsilon</code> parameter required for the solution of the CSP. It should be the same value used to compute the minimum number of samples required. This can be done with num_samples_eps_delta, but note that the number returned will be huge! In practice, fewer samples can be enough, but the actual number will strongly depend on the utility, in particular its variance.</p> <pre><code>from pydvl.utils import Dataset, Utility\nfrom pydvl.value import compute_shapley_values\n\nmodel = ...\ndata = Dataset(...)\nutility = Utility(model, data, score_range=(_min, _max))\nmin_iterations = num_samples_eps_delta(epsilon, delta, n, utility.score_range)\nvalues = compute_shapley_values(\n   u=utility, mode=\"group_testing\", n_iterations=min_iterations, eps=eps\n)\n</code></pre> <ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9\u21a9</p> </li> <li> <p>Okhrati, R., Lipani, A., 2021. A Multilinear Sampling Algorithm to Estimate Shapley Values, in: 2020 25th International Conference on Pattern Recognition (ICPR). Presented at the 2020 25th International Conference on Pattern Recognition (ICPR), IEEE, pp. 7992\u20137999. https://doi.org/10.1109/ICPR48806.2021.9412511 \u21a9</p> </li> <li> <p>Castro, J., G\u00f3mez, D., Tejada, J., 2009. Polynomial calculation of the Shapley value based on sampling. Computers   &amp; Operations Research, Selected papers presented at the Tenth International Symposium on Locational Decisions (ISOLDE X) 36, 1726\u20131730. https://doi.org/10.1016/j.cor.2008.04.004 \u21a9\u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Gurel, N.M., Li, B., Zhang, C., Spanos, C., Song, D., 2019. Efficient task-specific data valuation for nearest neighbor algorithms. Proc. VLDB Endow. 12, 1610\u20131623. https://doi.org/10.14778/3342263.3342637 \u21a9\u21a9</p> </li> </ol>"},{"location":"value/the-core/","title":"Core values","text":"<p>Shapley values define a fair way to distribute payoffs amongst all participants (training points) when they form a grand coalition, i.e. when the model is trained on the whole dataset. But they do not consider the question of stability: under which conditions do all participants in a game form the grand coalition? Are the payoffs distributed in such a way that prioritizes its formation?</p> <p>The Core is another solution concept in cooperative game theory that attempts to ensure stability in the sense that it provides the set of feasible payoffs that cannot be improved upon by a sub-coalition. This can be interesting for some applications of data valuation because it yields values consistent with training on the whole dataset, avoiding the spurious selection of subsets.</p> <p>It satisfies the following 2 properties:</p> <ul> <li> <p>Efficiency:   The payoffs are distributed such that it is not possible to make any   participant better off without making another one worse off.   \\(\\sum_{i \\in D} v(i) = u(D).\\)</p> </li> <li> <p>Coalitional rationality:   The sum of payoffs to the agents in any coalition \\(S\\) is at least as large as   the amount that these agents could earn by forming a coalition on their own.   \\(\\sum_{i \\in S} v(i) \\geq u(S), \\forall S \\subset D.\\)</p> </li> </ul> <p>The Core was first introduced into data valuation by (Yan and Procaccia, 2021)<sup>1</sup>, in the following form.</p>"},{"location":"value/the-core/#least-core-values","title":"Least Core values","text":"<p>Unfortunately, for many cooperative games the Core may be empty. By relaxing the coalitional rationality property by a subsidy \\(e \\gt 0\\), we are then able to find approximate payoffs:</p> \\[ \\sum_{i\\in S} v(i) + e \\geq u(S), \\forall S \\subset D, S \\neq \\emptyset \\ ,\\] <p>The Least Core (LC) values \\(\\{v\\}\\) for utility \\(u\\) are computed by solving the following linear program:</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; e &amp; \\\\ \\text{subject to} &amp; \\sum_{i\\in D} v(i) = u(D) &amp; \\\\ &amp; \\sum_{i\\in S} v(i) + e \\geq u(S) &amp;, \\forall S \\subset D, S \\neq \\emptyset  \\\\ \\end{array} \\] <p>Note that solving this program yields a set of solutions \\(\\{v_j:N \\rightarrow \\mathbb{R}\\}\\), whereas the Shapley value is a single function \\(v\\). In order to obtain a single valuation to use, one breaks ties by solving a quadratic program to select the \\(v\\) in the LC with the smallest \\(\\ell_2\\) norm. This is called the egalitarian least core.</p>"},{"location":"value/the-core/#exact-least-core","title":"Exact Least Core","text":"<p>This first algorithm is just a verbatim implementation of the definition, in compute_least_core_values. It computes all constraints for the linear problem by evaluating the utility on every subset of the training data, and returns as exact a value as the utility function allows (see what this means in Problems of Data Values).</p> <pre><code>from pydvl.value import compute_least_core_values\n\nvalues = compute_least_core_values(utility, mode=\"exact\")\n</code></pre>"},{"location":"value/the-core/#monte-carlo-least-core","title":"Monte Carlo Least Core","text":"<p>Because the number of subsets \\(S \\subseteq D \\setminus \\{i\\}\\) is \\(2^{ | D | - 1 }\\), one typically must resort to approximations.</p> <p>The simplest one consists in using a fraction of all subsets for the constraints.  (Yan and Procaccia, 2021)<sup>1</sup> show that a quantity of order \\(\\mathcal{O}((n - \\log \\Delta ) / \\delta^2)\\) is enough to obtain a so-called \\(\\delta\\)-approximate least core with high probability. I.e. the following property holds with probability \\(1-\\Delta\\) over the choice of subsets:</p> \\[ \\mathbb{P}_{S\\sim D}\\left[\\sum_{i\\in S} v(i) + e^{*} \\geq u(S)\\right] \\geq 1 - \\delta, \\] <p>where \\(e^{*}\\) is the optimal least core subsidy. This approximation is also implemented in compute_least_core_values:</p> <pre><code>from pydvl.value import compute_least_core_values\n\nvalues = compute_least_core_values(\n   utility, mode=\"montecarlo\", n_iterations=n_iterations\n)\n</code></pre> <p>Note</p> <p>Although any number is supported, it is best to choose <code>n_iterations</code> to be at least equal to the number of data points.</p> <p>Because computing the Least Core values requires the solution of a linear and a quadratic problem after computing all the utility values, we offer the possibility of splitting the latter from the former. This is useful when running multiple experiments: use mclc_prepare_problem to prepare a list of problems to solve, then solve them in parallel with lc_solve_problems.</p> <pre><code>from pydvl.value.least_core import mclc_prepare_problem, lc_solve_problems\n\nn_experiments = 10\nproblems = [mclc_prepare_problem(utility, n_iterations=n_iterations)\n    for _ in range(n_experiments)]\nvalues = lc_solve_problems(problems)\n</code></pre>"},{"location":"value/the-core/#method-comparison","title":"Method comparison","text":"<p>The TransferLab team reproduced the results of the original paper in a publication for the 2022 MLRC (Benmerzoug and Benito Delgado, 2023)<sup>2</sup>.</p> <p> Best sample removal on binary image classification </p> <p>Roughly speaking, MCLC performs better in identifying high value points, as measured by best-sample removal tasks. In all other aspects, it performs worse or similarly to TMCS at comparable sample budgets. But using an equal number of subsets is more computationally expensive because of the need to solve large linear and quadratic optimization problems.</p> <p> Worst sample removal on binary image classification </p> <p>For these reasons we recommend some variation of SV like TMCS for outlier detection, data cleaning and pruning, and perhaps MCLC for the selection of interesting points to be inspected for the improvement of data collection or model design.</p> <ol> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9\u21a9</p> </li> <li> <p>Benmerzoug, A., Benito Delgado, M. de, 2023. [Re] If you like Shapley, then you\u2019ll love the core. ReScience C 9. https://doi.org/10.5281/zenodo.8173733 \u21a9</p> </li> </ol>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"The python library for data valuation","text":"<p>pyDVL collects algorithms for data valuation and influence function computation. For the full list see Methods. It supports out-of-core and distributed computation, as well as local or distributed caching of results.</p> <p>If you're a first time user of pyDVL, we recommend you to go through Getting started.</p> <ul> <li> <p> Getting started</p> <p>Steps to install and requirements</p> </li> <li> <p> Example gallery</p> <p>Notebooks with worked-out examples of data valuation and influence functions</p> </li> <li> <p> Data valuation</p> <p>Basics of data valuation and description of the main algorithms</p> </li> <li> <p> Influence Function</p> <p>An introduction to the influence function and its computation with pyDVL</p> </li> <li> <p> Supported methods</p> <p>List of all methods implemented with references.</p> </li> <li> <p> API Reference</p> <p>Full documentation of the API</p> </li> </ul>"},{"location":"CHANGELOG/","title":"Changelog","text":""},{"location":"CHANGELOG/#unreleased","title":"Unreleased","text":""},{"location":"CHANGELOG/#fixed","title":"Fixed","text":"<ul> <li><code>FutureWarning</code> for <code>ParallelConfig</code> constantly raised without actually    instantiating the object   PR #562</li> </ul>"},{"location":"CHANGELOG/#090-new-methods-better-docs-and-bugfixes","title":"0.9.0 - \ud83c\udd95 New methods, better docs and bugfixes \ud83d\udcda\ud83d\udc1e","text":""},{"location":"CHANGELOG/#added","title":"Added","text":"<ul> <li>New method <code>MSR Banzhaf</code> with accompanying notebook, and new stopping   criterion <code>RankCorrelation</code> PR #520</li> <li>New method: <code>NystroemSketchInfluence</code> PR #504</li> <li>New preconditioned block variant of conjugate gradient   PR #507</li> <li>Improvements to documentation: fixes, links, text, example gallery, LFS and   more PR #532,    PR #543</li> <li>Glossary of data valuation and influence terms in the documentation   [PR #537](https://github.com/aai-institute/pyDVL/pull/537</li> <li>Documentation about writing notes for new features, changes or deprecations   PR #557</li> </ul>"},{"location":"CHANGELOG/#fixed_1","title":"Fixed","text":"<ul> <li>Bug in <code>LissaInfluence</code>, when not using CPU device   PR #495</li> <li>Memory issue with <code>CgInfluence</code> and <code>ArnoldiInfluence</code> PR #498</li> <li>Raising specific error message with install instruction, when trying to load    <code>pydvl.utils.cache.memcached</code> without <code>pymemcache</code> installed.   If <code>pymemcache</code> is available, all symbols from <code>pydvl.utils.cache.memcached</code>    are available through <code>pydvl.utils.cache</code> PR #509 </li> </ul>"},{"location":"CHANGELOG/#changed","title":"Changed","text":"<ul> <li>Add property <code>model_dtype</code> to instances of type <code>TorchInfluenceFunctionModel</code></li> <li>Bump versions of CI actions to avoid warnings   PR #502</li> <li>Add Python Version 3.11 to supported versions   PR #510</li> <li>Documentation improvements and cleanup   PR #521,   PR #522</li> <li>Simplified parallel backend configuration   PR #549</li> </ul>"},{"location":"CHANGELOG/#081-new-method-and-notebook-games-with-exact-shapley-values-bug-fixes-and-cleanup","title":"0.8.1 - \ud83c\udd95 \ud83c\udfd7  New method and notebook, Games with exact shapley values, bug fixes and cleanup","text":""},{"location":"CHANGELOG/#added_1","title":"Added","text":"<ul> <li>Implement new method: <code>EkfacInfluence</code> PR #451</li> <li>New notebook to showcase ekfac for LLMs   PR #483</li> <li>Implemented exact games in Castro et al. 2009 and 2017   PR #341</li> </ul>"},{"location":"CHANGELOG/#fixed_2","title":"Fixed","text":"<ul> <li>Bug in using <code>DaskInfluenceCalcualator</code> with <code>TorchnumpyConverter</code>   for single dimensional arrays    PR #485</li> <li>Fix implementations of <code>to</code> methods of <code>TorchInfluenceFunctionModel</code>    implementations PR #487</li> <li>Fixed bug with checking for converged values in semivalues   PR #341</li> </ul>"},{"location":"CHANGELOG/#changed_1","title":"Changed","text":"<ul> <li>Add applications of data valuation section, display examples more prominently,   make all sections visible in table of contents, use mkdocs material cards   in the home page PR #492</li> </ul>"},{"location":"CHANGELOG/#080-new-interfaces-scaling-computation-bug-fixes-and-improvements","title":"0.8.0 - \ud83c\udd95 New interfaces, scaling computation, bug fixes and improvements \ud83c\udf81","text":""},{"location":"CHANGELOG/#added_2","title":"Added","text":"<ul> <li>New cache backends: InMemoryCacheBackend and DiskCacheBackend   PR #458</li> <li>New influence function interface <code>InfluenceFunctionModel</code></li> <li>Data parallel computation with <code>DaskInfluenceCalculator</code> PR #26</li> <li>Sequential batch-wise computation and write to disk with    <code>SequentialInfluenceCalculator</code> PR #377</li> <li>Adapt notebooks to new influence abstractions   PR #430</li> </ul>"},{"location":"CHANGELOG/#changed_2","title":"Changed","text":"<ul> <li>Refactor and simplify caching implementation    PR #458</li> <li>Simplify display of computation progress   PR #466</li> <li>Improve readme and explain better the examples   PR #465</li> <li>Simplify and improve tests, add CodeCov code coverage   PR #429</li> <li>Breaking Changes</li> <li>Removed <code>compute_influences</code> and all related code.     Replaced by new <code>InfluenceFunctionModel</code> interface. Removed modules:<ul> <li>influence.general</li> <li>influence.inversion</li> <li>influence.twice_differentiable</li> <li>influence.torch.torch_differentiable</li> </ul> </li> </ul>"},{"location":"CHANGELOG/#fixed_3","title":"Fixed","text":"<ul> <li>Import bug in README PR #457</li> </ul>"},{"location":"CHANGELOG/#071-new-methods-bug-fixes-and-improvements-for-local-tests","title":"0.7.1 - \ud83c\udd95 New methods, bug fixes and improvements for local tests \ud83d\udc1e\ud83e\uddea","text":""},{"location":"CHANGELOG/#added_3","title":"Added","text":"<ul> <li>New method: Class-wise Shapley values   PR #338</li> <li>New method: Data-OOB by @BastienZim    PR #426,    PR $431</li> <li>Added <code>AntitheticPermutationSampler</code> PR #439</li> <li>Faster semi-value computation with per-index check of stopping criteria (optional)   PR #437</li> </ul>"},{"location":"CHANGELOG/#fixed_4","title":"Fixed","text":"<ul> <li>Fix initialization of <code>data_names</code> in <code>ValuationResult.zeros()</code> PR #443</li> </ul>"},{"location":"CHANGELOG/#changed_3","title":"Changed","text":"<ul> <li>No longer using docker within tests to start a memcached server   PR #444</li> <li>Using pytest-xdist for faster local tests   PR #440</li> <li>Improvements and fixes to notebooks   PR #436</li> <li>Refactoring of parallel module. Old imports will stop working in v0.9.0   PR #421</li> </ul>"},{"location":"CHANGELOG/#070-documentation-and-if-overhaul-new-methods-and-bug-fixes","title":"0.7.0 - \ud83d\udcda\ud83c\udd95 Documentation and IF overhaul, new methods and bug fixes \ud83d\udca5\ud83d\udc1e","text":"<p>This is our first \u03b2 release! We have worked hard to deliver improvements across the board, with a focus on documentation and usability. We have also reworked the internals of the <code>influence</code> module, improved parallelism and handling of randomness.</p>"},{"location":"CHANGELOG/#added_4","title":"Added","text":"<ul> <li>Implemented solving the Hessian equation via spectral low-rank approximation   PR #365</li> <li>Enabled parallel computation for Leave-One-Out values   PR #406</li> <li>Added more abbreviations to documentation   PR #415</li> <li>Added seed to functions from <code>pydvl.utils.numeric</code>, <code>pydvl.value.shapley</code> and   <code>pydvl.value.semivalues</code>. Introduced new type <code>Seed</code> and conversion function    <code>ensure_seed_sequence</code>.   PR #396</li> <li>Added <code>batch_size</code> parameter to <code>compute_banzhaf_semivalues</code>,   <code>compute_beta_shapley_semivalues</code>, <code>compute_shapley_semivalues</code> and   <code>compute_generic_semivalues</code>.   PR #428</li> <li>Added classwise Shapley as proposed by (Schoch et al. 2021)   [https://arxiv.org/abs/2211.06800]   PR #338</li> </ul>"},{"location":"CHANGELOG/#changed_4","title":"Changed","text":"<ul> <li>Replaced sphinx with mkdocs for documentation. Major overhaul of documentation   PR #352</li> <li>Made ray an optional dependency, relying on joblib as default parallel backend   PR #408</li> <li>Decoupled <code>ray.init</code> from <code>ParallelConfig</code> PR #373</li> <li>Breaking Changes</li> <li>Signature change: return information about Hessian inversion from     <code>compute_influence_factors</code> PR #375</li> <li>Major changes to IF interface and functionality. Foundation for a framework     abstraction for IF computation.     PR #278 PR #394</li> <li>Renamed <code>semivalues</code> to <code>compute_generic_semivalues</code> PR #413</li> <li>New <code>joblib</code> backend as default instead of ray. Simplify MapReduceJob.     PR #355</li> <li>Bump torch dependency for influence package to 2.0     PR #365</li> </ul>"},{"location":"CHANGELOG/#fixed_5","title":"Fixed","text":"<ul> <li>Fixes to parallel computation of generic semi-values: properly handle all   samplers and stopping criteria, irrespective of parallel backend.   PR #372</li> <li>Optimises memory usage in IF calculation   PR #375</li> <li>Fix adding valuation results with overlapping indices and different lengths   PR #370</li> <li>Fixed bugs in conjugate gradient and <code>linear_solve</code> PR #358</li> <li>Fix installation of dev requirements for Python3.10   PR #382</li> <li>Improvements to IF documentation   PR #371</li> </ul>"},{"location":"CHANGELOG/#061-bug-fixes-and-small-improvements","title":"0.6.1 - \ud83c\udfd7 Bug fixes and small improvements","text":"<ul> <li>Fix parsing keyword arguments of <code>compute_semivalues</code> dispatch function   PR #333</li> <li>Create new <code>RayExecutor</code> class based on the concurrent.futures API,   use the new class to fix an issue with Truncated Monte Carlo Shapley   (TMCS) starting too many processes and dying, plus other small changes   PR #329</li> <li>Fix creation of GroupedDataset objects using the <code>from_arrays</code>   and <code>from_sklearn</code> class methods    PR #324</li> <li>Fix release job not triggering on CI when a new tag is pushed   PR #331</li> <li>Added alias <code>ApproShapley</code> from Castro et al. 2009 for permutation Shapley   PR #332</li> </ul>"},{"location":"CHANGELOG/#060-new-algorithms-cleanup-and-bug-fixes","title":"0.6.0 - \ud83c\udd95 New algorithms, cleanup and bug fixes \ud83c\udfd7","text":"<ul> <li>Fixes in <code>ValuationResult</code>: bugs around data names, semantics of   <code>empty()</code>, new method <code>zeros()</code> and normalised random values   PR #327</li> <li>New method: Implements generalised semi-values for data valuation,   including Data Banzhaf and Beta Shapley, with configurable sampling strategies   PR #319</li> <li>Adds kwargs parameter to <code>from_array</code> and <code>from_sklearn</code> Dataset and   GroupedDataset class methods   PR #316</li> <li>PEP-561 conformance: added <code>py.typed</code> PR #307</li> <li>Removed default non-negativity constraint on least core subsidy   and added instead a <code>non_negative_subsidy</code> boolean flag.   Renamed <code>options</code> to <code>solver_options</code> and pass it as dict.   Change default least-core solver to SCS with 10000 max_iters.   PR #304</li> <li>Cleanup: removed unnecessary decorator <code>@unpackable</code> PR #233</li> <li>Stopping criteria: fixed problem with <code>StandardError</code> and enable proper   composition of index convergence statuses. Fixed a bug with <code>n_jobs</code> in   <code>truncated_montecarlo_shapley</code>.   PR #300 and   PR #305</li> <li>Shuffling code around to allow for simpler user imports, some cleanup and   documentation fixes.   PR #284</li> <li>Bug fix: Warn instead of raising an error when <code>n_iterations</code>   is less than the size of the dataset in Monte Carlo Least Core   PR #281</li> </ul>"},{"location":"CHANGELOG/#050-fixes-nicer-interfaces-and-more-breaking-changes","title":"0.5.0 - \ud83d\udca5 Fixes, nicer interfaces and... more breaking changes \ud83d\ude12","text":"<ul> <li>Fixed parallel and antithetic Owen sampling for Shapley values. Simplified   and extended tests.   PR #267</li> <li>Added <code>Scorer</code> class for a cleaner interface. Fixed minor bugs around   Group-Testing Shapley, added more tests and switched to cvxpy for the solver.   PR #264</li> <li>Generalised stopping criteria for valuation algorithms. Improved classes   <code>ValuationResult</code> and <code>Status</code> with more operations. Some minor issues fixed.   PR #252</li> <li>Fixed a bug whereby <code>compute_shapley_values</code> would only spawn one process when   using <code>n_jobs=-1</code> and Monte Carlo methods.   PR #270</li> <li>Bugfix in <code>RayParallelBackend</code>: wrong semantics for <code>kwargs</code>.   PR #268</li> <li>Splitting of problem preparation and solution in Least-Core computation.   Umbrella function for LC methods.   PR #257 </li> <li>Operations on <code>ValuationResult</code> and <code>Status</code> and some cleanup   PR #248</li> <li>Bug fix and minor improvements: Fixes bug in TMCS with remote Ray cluster,   raises an error for dummy sequential parallel backend with TMCS, clones model   inside <code>Utility</code> before fitting by default, with flag <code>clone_before_fit</code>    to disable it, catches all warnings in <code>Utility</code> when <code>show_warnings</code> is    <code>False</code>. Adds Miner and Gloves toy games utilities   PR #247</li> </ul>"},{"location":"CHANGELOG/#040-new-algorithms-and-more-breaking-changes","title":"0.4.0 - \ud83c\udfed\ud83d\udca5 New algorithms and more breaking changes","text":"<ul> <li>GH action to mark issues as stale   PR #201</li> <li>Disabled caching of Utility values as well as repeated evaluations by default   PR #211</li> <li>Test and officially support Python version 3.9 and 3.10    PR #208</li> <li>Breaking change: Introduces a class ValuationResult to gather and inspect   results from all valuation algorithms   PR #214</li> <li>Fixes bug in Influence calculation with multidimensional input and adds new   example notebook   PR #195</li> <li>Breaking change: Passes the input to <code>MapReduceJob</code> at initialization,   removes <code>chunkify_inputs</code> argument from <code>MapReduceJob</code>, removes <code>n_runs</code>   argument from <code>MapReduceJob</code>, calls the parallel backend's <code>put()</code> method for   each generated chunk in <code>_chunkify()</code>, renames ParallelConfig's <code>num_workers</code>   attribute to <code>n_local_workers</code>, fixes a bug in <code>MapReduceJob</code>'s chunkification   when <code>n_runs</code> &gt;= <code>n_jobs</code>, and defines a sequential parallel backend to run   all jobs in the current thread   PR #232</li> <li>New method: Implements exact and monte carlo Least Core for data valuation,   adds <code>from_arrays()</code> class method to the <code>Dataset</code> and <code>GroupedDataset</code>   classes, adds <code>extra_values</code> argument to <code>ValuationResult</code>, adds   <code>compute_removal_score()</code> and <code>compute_random_removal_score()</code> helper functions   PR #237</li> <li>New method: Group Testing Shapley for valuation, from Jia et al. 2019 PR #240</li> <li>Fixes bug in ray initialization in <code>RayParallelBackend</code> class   PR #239</li> <li>Implements \"Egalitarian Least Core\", adds cvxpy as a   dependency and uses it instead of scipy as optimizer   PR #243</li> </ul>"},{"location":"CHANGELOG/#030-breaking-changes","title":"0.3.0 - \ud83d\udca5 Breaking changes","text":"<ul> <li>Simplified and fixed powerset sampling and testing   PR #181</li> <li>Simplified and fixed publishing to PyPI from CI   PR #183</li> <li>Fixed bug in release script and updated contributing docs.   PR #184</li> <li>Added Pull Request template   PR #185</li> <li>Modified Pull Request template to automatically link PR to issue   PR ##186</li> <li>First implementation of Owen Sampling, squashed scores, better testing   PR #194</li> <li>Improved documentation on caching, Shapley, caveats of values, bibtex   PR #194</li> <li>Breaking change: Rearranging of modules to accommodate for new methods   PR #194</li> </ul>"},{"location":"CHANGELOG/#020-better-docs","title":"0.2.0 - \ud83d\udcda Better docs","text":"<p>Mostly API documentation and notebooks, plus some bugfixes.</p>"},{"location":"CHANGELOG/#added_5","title":"Added","text":"<p>In PR #161: - Support for $$ math in sphinx docs. - Usage of sphinx extension for external links (introducing new directives like   <code>:gh:</code>, <code>:issue:</code> and <code>:tfl:</code> to construct standardised links to external   resources). - Only update auto-generated documentation files if there are changes. Some   minor additions to <code>update_docs.py</code>. - Parallelization of exact combinatorial Shapley. - Integrated KNN shapley into the main interface <code>compute_shapley_values</code>.</p>"},{"location":"CHANGELOG/#changed_5","title":"Changed","text":"<p>In PR #161: - Improved main docs and Shapley notebooks. Added or fixed many docstrings,   readme and documentation for contributors. Typos, grammar and style in code,   documentation and notebooks. - Internal renaming and rearranging in the parallelization and caching modules.</p>"},{"location":"CHANGELOG/#fixed_6","title":"Fixed","text":"<ul> <li>Bug in random matrix generation   PR #161.</li> <li>Bugs in MapReduceJob's <code>_chunkify</code> and <code>_backpressure</code> methods   PR #176.</li> </ul>"},{"location":"CHANGELOG/#010-first-release","title":"0.1.0 - \ud83c\udf89 first release","text":"<p>This is very first release of pyDVL.</p> <p>It contains:</p> <ul> <li> <p>Data Valuation Methods:</p> </li> <li> <p>Leave-One-Out</p> </li> <li>Influence Functions</li> <li>Shapley:<ul> <li>Exact Permutation and Combinatorial</li> <li>Montecarlo Permutation and Combinatorial</li> <li>Truncated Montecarlo Permutation</li> </ul> </li> <li>Caching of results with Memcached</li> <li>Parallelization of computations with Ray</li> <li>Documentation</li> <li>Notebooks containing examples of different use cases</li> </ul>"},{"location":"CONTRIBUTING/","title":"Contributing to pyDVL","text":"<p>The goal of pyDVL is to be a repository of successful algorithms for the valuation of data, in a broader sense. Contributions are welcome from anyone in the form of pull requests, bug reports and feature requests.</p> <p>We will consider for inclusion any (tested) implementation of an algorithm appearing in a peer-reviewed journal (even if the method does not improve the state of the art, for benchmarking and comparison purposes). We are also open to improvements to the currently implemented methods and other ideas. Please open a ticket with yours.</p> <p>If you are interested in setting up a similar project, consider the template  pymetrius.</p>"},{"location":"CONTRIBUTING/#local-development","title":"Local development","text":"<p>This project uses black to format code and pre-commit to invoke it as a git pre-commit hook. Consider installing any of black's IDE integrations to make your life easier.</p> <p>Run the following to set up the pre-commit git hook to run before pushes:</p> <pre><code>pre-commit install --hook-type pre-push\n</code></pre> <p>Additionally, we use Git LFS for some files like images. Install with</p> <pre><code>git lfs install\n</code></pre>"},{"location":"CONTRIBUTING/#setting-up-your-environment","title":"Setting up your environment","text":"<p>We strongly suggest using some form of virtual environment for working with the library. E.g. with venv:</p> <pre><code>python -m venv ./venv\n. venv/bin/activate  # `venv\\Scripts\\activate` in windows\npip install -r requirements-dev.txt -r requirements-docs.txt\n</code></pre> <p>With conda:</p> <pre><code>conda create -n pydvl python=3.8\nconda activate pydvl\npip install -r requirements-dev.txt -r requirements-docs.txt\n</code></pre> <p>A very convenient way of working with your library during development is to install it in editable mode into your environment by running</p> <pre><code>pip install -e .\n</code></pre> <p>In order to build the documentation locally (which is done as part of the tox suite) you need to install additional non-python dependencies as described in the documentation of mkdocs-material.</p> <p>In addition, pandoc is required. Except for OSX,  it should be installed automatically as a dependency with  <code>requirements-docs.txt</code>. Under OSX you can install pandoc  (you'll need at least version 2.11) with:</p> <pre><code>brew install pandoc\n</code></pre> <p>Remember to mark all autogenerated directories as excluded in your IDE. In particular <code>docs_build</code> and <code>.tox</code> should be marked as excluded to avoid slowdowns when searching or refactoring code.</p> <p>If you use remote execution, don't forget to exclude data paths from deployment (unless you really want to sync them).</p>"},{"location":"CONTRIBUTING/#testing","title":"Testing","text":"<p>Automated builds, tests, generation of documentation and publishing are handled by CI pipelines. Before pushing your changes to the remote we recommend to execute <code>tox</code> locally in order to detect mistakes early on and to avoid failing pipelines. tox will: * run the test suite * build the documentation * build and test installation of the package. * generate coverage and pylint reports in html, as well as badges.</p> <p>You can configure pytest, coverage and pylint by adjusting pyproject.toml.</p> <p>Besides the usual unit tests, most algorithms are tested using pytest. This requires ray for the parallelization and Memcached for caching. Please install both before running the tests. We run tests in CI as well.</p> <p>It is possible to pass optional command line arguments to pytest, for example to run only certain tests using patterns (<code>-k</code>) or marker (<code>-m</code>).</p> <pre><code>tox -e tests -- &lt;optional arguments&gt;\n</code></pre> <p>There are a few important arguments:</p> <ul> <li><code>--memcached-service</code> allows to change the default of <code>localhost:11211</code> (memcached's default)   to a different address.</li> </ul> <p>Memcached is needed for testing   caching as well as speeding certain methods (e.g. Permutation Shapley).</p> <p>To start memcached locally in the background with Docker use:</p> <pre><code>docker run --name pydvl-memcache -p 11211:11211 -d memcached\n</code></pre> <ul> <li><code>-n</code> sets the number of parallel workers for    pytest-xdist.</li> </ul> <p>There are two layers of parallelization in the tests.   An inner one within the tests themselves, i.e. the parallelism in the algorithms,   and an outer one by pytest-xdist. The latter is controlled by the <code>-n</code> argument.   If you experience segmentation faults with the tests,   try running them with <code>-n 0</code> to disable parallelization.</p> <ul> <li><code>--slow-tests</code> enables running slow tests. See below for a description   of slow tests.</li> </ul>"},{"location":"CONTRIBUTING/#markers","title":"Markers","text":"<p>We use a few different markers to differentiate between tests and runs groups of them of separately. Use <code>pytest --markers</code> to get a list and description of all available markers.</p> <p>Two important markers are:</p> <ul> <li><code>pytest.mark.slow</code> which is used to mark slow tests and skip them by default.</li> </ul> <p>A slow test is any test that takes 45 seconds or more to run and that can be   skipped most of the time. In some cases a test is slow, but it is required   in order to ensure that a feature works as expected and that are no bugs.   In those cases, we should not use this marker.</p> <p>Slow tests are always run on CI. Locally, they are skipped   by default but can be additionally run using: <code>pytest --slow-tests</code>.</p> <ul> <li><code>pytest.mark.torch</code> which is used to mark tests that require PyTorch.</li> </ul> <p>To test modules that rely on PyTorch, use:</p> <pre><code>tox -e tests -- -m \"torch\"\n</code></pre>"},{"location":"CONTRIBUTING/#other-things","title":"Other Things","text":"<p>To test the notebooks separately, run (see below for details):</p> <pre><code>tox -e notebook-tests\n</code></pre> <p>To create a package locally, run: <pre><code>python setup.py sdist bdist_wheel\n</code></pre></p>"},{"location":"CONTRIBUTING/#notebooks","title":"Notebooks","text":"<p>We use notebooks both as documentation (copied over to <code>docs/examples</code>) and as integration tests. All notebooks in the <code>notebooks</code> directory are executed during the test run. Because run times are typically too long for large datasets, you must check for the <code>CI</code> environment variable to work with smaller ones. For example, you can select a subset of the data:</p> <pre><code># In CI we only use a subset of the training set\nif os.environ.get('CI'):\n    training_data = training_data[:10]\n</code></pre> <p>This switching should happen in a separate notebook cell tagged with <code>hide</code> to hide the cell's input and output when rendering it as part of the documents. We want to avoid as much clutter and boilerplate as possible in the notebooks themselves.</p> <p>Because we want documentation to include the full dataset, we commit notebooks with their outputs running with full datasets to the repo. The notebooks are then added by CI to the section Examples of the documentation.</p>"},{"location":"CONTRIBUTING/#hiding-cells-in-notebooks","title":"Hiding cells in notebooks","text":"<p>Switching between CI or not, importing generic modules and plotting results are all examples of boilerplate code irrelevant to a reader interested in pyDVL's functionality. For this reason we choose to isolate this code into separate cells which are then hidden in the documentation.</p> <p>In order to do this, cells are marked with tags understood by the mkdocs plugin <code>mkdocs-jupyter</code>, namely adding the following to the metadata of the relevant cells:</p> <pre><code>\"tags\": [\n  \"hide\"\n]\n</code></pre> <p>To hide the cell's input and output.</p> <p>Or:</p> <pre><code>\"tags\": [\n  \"hide-input\"\n]\n</code></pre> <p>To only hide the input and</p> <p><pre><code>\"tags\": [\n  \"hide-output\"\n]\n</code></pre> for hiding the output only.</p> <p>It is important to leave a warning at the top of the document to avoid confusion. Examples for hidden imports and plots are available in the notebooks, e.g. in notebooks/shapley_basic_spotify.ipynb.</p>"},{"location":"CONTRIBUTING/#plots-in-notebooks","title":"Plots in Notebooks","text":"<p>If you add a plot to a notebook, which should also render nicely in browser dark mode, add the tag invertible-output, i.e.</p> <p><pre><code>\"tags\": [\n  \"invertible-output\"\n]\n</code></pre> This applies a simple CSS-filter to the output image of the cell.</p>"},{"location":"CONTRIBUTING/#documentation","title":"Documentation","text":"<p>API documentation and examples from notebooks are built with mkdocs, using a number of plugins, including mkdoctrings, with versioning handled by mike.</p> <p>Notebooks are an integral part of the documentation as well, please read the section on notebooks above.</p> <p>If you want to build the documentation locally, please make sure you followed the instructions in the section  Setting up your environment.</p> <p>Use the following command to build the documentation the same way it is done in CI:</p> <pre><code>mkdocs build\n</code></pre> <p>Locally, you can use this command instead to continuously rebuild documentation on changes to the <code>docs</code> and <code>src</code> folder:</p> <pre><code>mkdocs serve\n</code></pre> <p>This will rebuild the documentation on changes to <code>.md</code> files inside <code>docs</code>, notebooks and python files.</p> <p>On OSX, it is possible that the cairo lib file is not properly linked when installed via homebrew. In this case you might encounter an error like this <pre><code>OSError: no library called \"cairo-2\" was found\nno library called \"cairo\" was found\nno library called \"libcairo-2\" was found\n</code></pre> when calling <code>mkdocs build</code> or <code>mkdocs serve</code>. This can be resolved via setting the environment variable <code>DYLD_FALLBACK_LIBRARY_PATH</code>: <pre><code>export DYLD_FALLBACK_LIBRARY_PATH=$DYLD_FALLBACK_LIBRARY_PATH:/opt/homebrew/lib\n</code></pre></p>"},{"location":"CONTRIBUTING/#adding-new-pages","title":"Adding new pages","text":"<p>Navigation is configured in <code>mkdocs.yaml</code> using the nav section. We use the plugin mkdoc-literate-nav which allows fine-grained control of the navigation structure. However, most pages are explicitly listed and manually arranged in the <code>nav</code> section of the configuration.</p>"},{"location":"CONTRIBUTING/#creating-stable-references-for-autorefs","title":"Creating stable references for autorefs","text":"<p>mkdocstrings includes the plugin autorefs to enable automatic linking across pages with e.g. <code>[a link][to-something]</code>. Anchors are autogenerated from section titles, and are not guaranteed to be unique. In order to ensure that a link will remain valid, add a custom anchor to the section title:</p> <pre><code>## Some section { #permanent-anchor-to-some-section }\n</code></pre> <p>(note the space after the opening brace). You can then refer to it within another markdown file with <code>[Some section][permanent-anchor-to-some-section]</code>.</p>"},{"location":"CONTRIBUTING/#adding-notes-about-new-features-changes-or-deprecations","title":"Adding notes about new features, changes or deprecations","text":"<p>We use the admonition extension of Mkdocs Material to create admonitions, also known as call-outs, that hold information about when a certain feature was added, changed or deprecated and optionally a description with more details. We put the admonition directly in a module's, a function's or class' docstring.</p> <p>We use the following syntax:</p> <pre><code>!!! tip \"&lt;Event Type&gt; in version &lt;Version Number&gt;\"\n\n    &lt;Optional Description&gt;\n</code></pre> <p>The description is useful when the note is about a smaller change such as a parameter.</p> <ul> <li>For a new feature, we use:</li> </ul> <pre><code>!!! tip \"New in version &lt;Version Number&gt;\"\n\n    &lt;Optional Description&gt;\n</code></pre> <ul> <li>For a change to an existing feature we use:</li> </ul> <pre><code>!!! tip \"Changed in version &lt;Version Number&gt;\"\n\n    &lt;Optional Description&gt;\n</code></pre> <p>For example, for a change in version <code>1.2.3</code> that adds kwargs   to a class' constructor we would write:</p> <pre><code>!!! tip \"Changed in version 1.2.3\"\n\n    Added kwargs to the constructor.\n</code></pre> <ul> <li>For a deprecation we use:</li> </ul> <pre><code>!!! tip \"Deprecated in version &lt;Version Number&gt;\"\n\n    &lt;Optional Description&gt;\n</code></pre>"},{"location":"CONTRIBUTING/#using-bibliography","title":"Using bibliography","text":"<p>Bibliographic citations are managed with the plugin mkdocs-bibtex. To enter a citation first add the entry to <code>docs/pydvl.bib</code>. For team contributor this should be an export of the Zotero folder <code>software/pydvl</code> in the TransferLab Zotero library. All other contributors just add the bibtex data, and a maintainer will add it to the group library upon merging.</p> <p>To add a citation inside a markdown file, use the notation <code>[@citekey]</code>. Alas, because of when mkdocs-bibtex enters the pipeline, it won't process docstrings. For module documentation, we manually inject html into the markdown files. For example, in <code>pydvl.value.shapley.montecarlo</code> we have:</p> <pre><code>\"\"\"\nModule docstring...\n\n## References\n\n[^1]: &lt;a name=\"ghorbani_data_2019\"&gt;&lt;/a&gt;Ghorbani, A., Zou, J., 2019.\n    [Data Shapley: Equitable Valuation of Data for Machine\n    Learning](https://proceedings.mlr.press/v97/ghorbani19c.html).\n    In: Proceedings of the 36th International Conference on Machine Learning,\n    PMLR, pp. 2242\u20132251.\n\"\"\"\n</code></pre> <p>and then later in the file, inside a function's docstring:</p> <pre><code>    This function implements (Ghorbani and Zou, 2019)&lt;sup&gt;&lt;a \n    href=\"#ghorbani_data_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n</code></pre>"},{"location":"CONTRIBUTING/#writing-mathematics","title":"Writing mathematics","text":"<p>Use LaTeX delimiters <code>$</code> and <code>$$</code> for inline and displayed mathematics respectively.</p> <p>Warning: backslashes must be escaped in docstrings! (although there are exceptions). For simplicity, declare the string as \"raw\" with the prefix <code>r</code>:</p> <pre><code># This will work\ndef f(x: float) -&gt; float:\n    r\"\"\" Computes \n    $${ f(x) = \\frac{1}{x^2} }$$\n    \"\"\"\n    return 1/(x*x)\n\n# This throws an obscure error\ndef f(x: float) -&gt; float:\n    \"\"\" Computes \n    $$\\frac{1}{x^2}$$\n    \"\"\"\n    return 1/(x*x)\n</code></pre> <p>Note how there is no space after the dollar signs. This is important! You can use braces for legibility like in the first example.</p>"},{"location":"CONTRIBUTING/#abbreviations","title":"Abbreviations","text":"<p>We keep the abbreviations used in the documentation inside the docs_include/abbreviations.md file.</p> <p>The syntax for abbreviations is:</p> <pre><code>*[ABBR]: Abbreviation\n</code></pre>"},{"location":"CONTRIBUTING/#ci","title":"CI","text":"<p>We use workflows to:</p> <ul> <li>Run the tests.</li> <li>Publish documentation.</li> <li>Publish packages to testpypi / pypi.</li> <li>Mark issues as stale after 30 days. We do this only for issues with the label   <code>awaiting-reply</code>   which indicates that we have answered a question / feature request / PR and   are waiting for the OP to reply / update his work.</li> </ul>"},{"location":"CONTRIBUTING/#tests","title":"Tests","text":"<p>We test all algorithms with simple datasets in CI jobs. This can amount to a sizeable amount of time, so care must be taken not to overdo it: 1. All algorithm tests must be on very simple datasets and as quick as possible 2. We try not to trigger CI pipelines when unnecessary (see Skipping CI runs). 3. We split the tests based on their duration into groups and run them in parallel.</p> <p>For that we use pytest-split    to first store the duration of all tests with    <code>tox -e tests -- --store-durations --slow-tests</code>    in a <code>.test_durations</code> file.</p> <p>Alternatively, we case use pytest directly    <code>pytest --store-durations --slow-tests</code>.</p> <p>Note This does not have to be done each time a new test or test case is added. For new tests and test cases pytes-split assumes average test execution time(calculated based on the stored information) for every test which does not have duration information stored. Thus, there's no need to store durations after changing the test suite. However, when there are major changes in the suite compared to what's stored in .test_durations, it's recommended to update the duration information with <code>--store-durations</code> to ensure that the splitting is in balance.</p> <p>Then we can have as many splits as we want:</p> <pre><code>tox -e tests -- --splits 3 --group 1\ntox -e tests -- --splits 3 --group 2\ntox -e tests -- --splits 3 --group 3\n</code></pre> <p>Alternatively, we case use pytest directly    <code>pytest --splits 3 ---group 1</code>.</p> <p>Each one of these commands should be run in a separate shell/job    to run the test groups in parallel and decrease the total runtime.</p>"},{"location":"CONTRIBUTING/#running-github-actions-locally","title":"Running Github Actions locally","text":"<p>To run Github Actions locally we use act. It uses the workflows defined in <code>.github/workflows</code> and determines the set of actions that need to be run. It uses the Docker API to either pull or build the necessary images, as defined in our workflow files and finally determines the execution path based on the dependencies that were defined.</p> <p>Once it has the execution path, it then uses the Docker API to run containers for each action based on the images prepared earlier. The environment variables  and filesystem are all configured to match what GitHub provides.</p> <p>You can install it manually using:</p> <pre><code>curl -s https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash -s -- -d -b ~/bin \n</code></pre> <p>And then simply add it to your PATH variable: <code>PATH=~/bin:$PATH</code></p> <p>Refer to its official readme for more installation options.</p>"},{"location":"CONTRIBUTING/#act-cheatsheet","title":"act cheatsheet","text":"<p>By default, <code>act</code> will run all  workflows in <code>.github/workflows</code>. You can use the <code>-W</code> flag to specify a specific workflow file to run, or you can rely on the job id to be unique (but then you'll see warnings for the workflows without that job id).</p> <pre><code># Run only the main tests for python 3.8 after a push event (implicit) \nact -W .github/workflows/run-tests-workflow.yaml \\\n    -j run-tests \\\n    --input tests_to_run=base\\\n    --input python_version=3.8\n</code></pre> <p>Other common flags are: </p> <pre><code># List all actions for all events:\nact -l\n\n# List the actions for a specific event:\nact workflow_dispatch -l\n\n# List the actions for a specific job:\nact -j lint -l\n\n# Run the default (`push`) event:\nact\n\n# Run a specific event:\nact pull_request\n\n# Run a specific job:\nact -j lint\n\n# Collect artifacts to the /tmp/artifacts folder:\nact --artifact-server-path /tmp/artifacts\n\n# Run a job in a specific workflow (useful if you have duplicate job names)\nact -j lint -W .github/workflows/tox.yml\n\n# Run in dry-run mode:\nact -n\n\n# Enable verbose-logging (can be used with any of the above commands)\nact -v\n</code></pre>"},{"location":"CONTRIBUTING/#example","title":"Example","text":"<p>To run the <code>publish</code> job (the most difficult one to test) you would simply use:</p> <ul> <li>When triggered by a release:</li> </ul> <pre><code>act release -j publish --eventpath events.json\n</code></pre> <p>With <code>events.json</code> containing:</p> <pre><code>{\n  \"act\": true\n}\n</code></pre> <p>This will use your current branch. If you want to test a specific branch   you have to use the <code>workflow_dispatch</code> event (see below).</p> <ul> <li>To instead run it as if it had been manually triggered (i.e. <code>workflow_dispatch</code>)   you would instead use:</li> </ul> <pre><code>act workflow_dispatch -j publish --eventpath events.json\n</code></pre> <p>With <code>events.json</code> containing:</p> <pre><code>{\n  \"act\": true,\n  \"inputs\": {\n    \"tag_name\": \"v0.6.0\"\n  }\n}\n</code></pre>"},{"location":"CONTRIBUTING/#skipping-ci-runs","title":"Skipping CI runs","text":"<p>One sometimes would like to skip CI for certain commits (e.g. updating the readme). In order to do this, simply prefix the commit message with <code>[skip ci]</code>. The string can be anywhere, but adding it to the beginning of the commit message makes it more evident when looking at commits in a PR.</p> <p>Refer to the official GitHub documentation  for more information.</p>"},{"location":"CONTRIBUTING/#release-processes","title":"Release processes","text":""},{"location":"CONTRIBUTING/#automatic-release-process","title":"Automatic release process","text":"<p>In order to create an automatic release, a few prerequisites need to be satisfied:</p> <ul> <li>The project's virtualenv needs to be active</li> <li>The repository needs to be on the <code>develop</code> branch</li> <li>The repository must be clean (including no untracked files)</li> </ul> <p>Then, a new release can be created using the script <code>build_scripts/release-version.sh</code> (leave out the version parameter to have <code>bumpversion</code> automatically derive the next release version by bumping the patch part):</p> <pre><code>build_scripts/release-version.sh 0.1.6\n</code></pre> <p>To find out how to use the script, pass the <code>-h</code> or <code>--help</code> flags:</p> <pre><code>build_scripts/release-version.sh --help\n</code></pre> <p>If running in interactive mode (without <code>-y|--yes</code>), the script will output a summary of pending changes and ask for confirmation before executing the actions.</p> <p>Once this is done, a tag will be created on the repository. You should then create a GitHub release for that tag. That will a trigger a CI pipeline that will automatically create a package and publish it from CI to PyPI.</p>"},{"location":"CONTRIBUTING/#manual-release-process","title":"Manual release process","text":"<p>If the automatic release process doesn't cover your use case, you can also create a new release manually by following these steps:</p> <ol> <li>(Repeat as needed) implement features on feature branches merged into   <code>develop</code>. Each merge into develop will publish a new pre-release version     to TestPyPI. These versions can be installed using <code>pip install --pre     --index-url https://test.pypi.org/simple/</code>.</li> <li>When ready to release: From the develop branch create the release branch and    perform release activities (update changelog, news, ...). For your own    convenience, define an env variable for the release version     <pre><code>export RELEASE_VERSION=\"vX.Y.Z\"\ngit checkout develop\ngit branch release/${RELEASE_VERSION} &amp;&amp; git checkout release/${RELEASE_VERSION}\n</code></pre></li> <li>Run <code>bumpversion --commit release</code> if the release is only a patch release,    otherwise the full version can be specified using     <code>bumpversion --commit --new-version X.Y.Z release</code>    (the <code>release</code> part is ignored but required by bumpversion ).</li> <li>Merge the release branch into <code>master</code>, tag the merge commit, and push back to the repo.     The CI pipeline publishes the package based on the tagged commit.     <pre><code>git checkout master\ngit merge --no-ff release/${RELEASE_VERSION}\ngit tag -a ${RELEASE_VERSION} -m\"Release ${RELEASE_VERSION}\"\ngit push --follow-tags origin master\n</code></pre></li> <li>Switch back to the release branch <code>release/vX.Y.Z</code> and pre-bump the version:    <code>bumpversion --commit patch</code>. This ensures that <code>develop</code> pre-releases are    always strictly more recent than the last published release version from     <code>master</code>.</li> <li>Merge the release branch into <code>develop</code>:     <pre><code>git checkout develop\ngit merge --no-ff release/${RELEASE_VERSION}\ngit push origin develop\n</code></pre></li> <li>Delete the release branch if necessary:     <code>git branch -d release/${RELEASE_VERSION}</code></li> <li>Create a GitHub    release    for the created tag.</li> <li>Pour yourself a cup of coffee, you earned it!  </li> <li>A package will be automatically created and published from CI to PyPI.</li> </ol>"},{"location":"CONTRIBUTING/#ci-and-requirements-for-publishing","title":"CI and requirements for publishing","text":"<p>In order to publish new versions of the package from the development branch, the CI pipeline requires the following secret variables set up:</p> <pre><code>TEST_PYPI_USERNAME\nTEST_PYPI_PASSWORD\nPYPI_USERNAME\nPYPI_PASSWORD\n</code></pre> <p>The first 2 are used after tests run on the develop branch's CI workflow  to automatically publish packages to TestPyPI.</p> <p>The last 2 are used in the publish.yaml CI workflow to publish packages to PyPI from <code>develop</code> after a GitHub release.</p>"},{"location":"CONTRIBUTING/#publish-to-testpypi","title":"Publish to TestPyPI","text":"<p>We use bump2version to bump the build part of the version number without commiting or tagging the change and then publish a package to TestPyPI from CI using Twine. The version has the GitHub run number appended. </p> <p>For more details refer to the files .github/workflows/publish.yaml and .github/workflows/tox.yaml.</p>"},{"location":"api/pydvl/","title":"API Reference","text":""},{"location":"api/pydvl/#pydvl","title":"pydvl","text":""},{"location":"api/pydvl/#pydvl--the-python-data-valuation-library-api","title":"The Python Data Valuation Library API","text":"<p>This is the API documentation for the Python Data Valuation Library (PyDVL). Use the table of contents to access the documentation for each module.</p> <p>The two main modules you will want to look at are value and influence.</p>"},{"location":"api/pydvl/influence/","title":"Influence","text":""},{"location":"api/pydvl/influence/#pydvl.influence","title":"pydvl.influence","text":"<p>This package contains algorithms for the computation of the influence function.</p> <p>See The Influence function for an introduction to the concepts and methods implemented here.</p> <p>Warning</p> <p>Much of the code in this package is experimental or untested and is subject to modification. In particular, the package structure and basic API will probably change.</p>"},{"location":"api/pydvl/influence/array/","title":"Array","text":""},{"location":"api/pydvl/influence/array/#pydvl.influence.array","title":"pydvl.influence.array","text":"<p>This module provides classes and utilities for handling large arrays that are chunked and lazily evaluated. It includes abstract base classes for converting between tensor types and NumPy arrays, aggregating blocks of data, and abstract representations of lazy arrays. Concrete implementations are provided for handling chunked lazy arrays (chunked in one resp. two dimensions), with support for efficient storage and retrieval using the Zarr library.</p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NumpyConverter","title":"NumpyConverter","text":"<p>             Bases: <code>Generic[TensorType]</code>, <code>ABC</code></p> <p>Base class for converting TensorType objects into numpy arrays and vice versa.</p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NumpyConverter.to_numpy","title":"to_numpy  <code>abstractmethod</code>","text":"<pre><code>to_numpy(x: TensorType) -&gt; NDArray\n</code></pre> <p>Override this method for converting a TensorType object into a numpy array</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>@abstractmethod\ndef to_numpy(self, x: TensorType) -&gt; NDArray:\n    \"\"\"Override this method for converting a TensorType object into a numpy array\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NumpyConverter.from_numpy","title":"from_numpy  <code>abstractmethod</code>","text":"<pre><code>from_numpy(x: NDArray) -&gt; TensorType\n</code></pre> <p>Override this method for converting a numpy array into a TensorType object</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>@abstractmethod\ndef from_numpy(self, x: NDArray) -&gt; TensorType:\n    \"\"\"Override this method for converting a numpy array into a TensorType object\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.SequenceAggregator","title":"SequenceAggregator","text":"<p>             Bases: <code>Generic[TensorType]</code>, <code>ABC</code></p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.SequenceAggregator.__call__","title":"__call__  <code>abstractmethod</code>","text":"<pre><code>__call__(tensor_generator: Generator[TensorType, None, None])\n</code></pre> <p>Aggregates tensors from a generator.</p> <p>Implement this method to define how a sequence of tensors, provided by a generator, should be combined.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>@abstractmethod\ndef __call__(self, tensor_generator: Generator[TensorType, None, None]):\n    \"\"\"\n    Aggregates tensors from a generator.\n\n    Implement this method to define how a sequence of tensors, provided by a\n    generator, should be combined.\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.ListAggregator","title":"ListAggregator","text":"<p>             Bases: <code>SequenceAggregator</code></p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.ListAggregator.__call__","title":"__call__","text":"<pre><code>__call__(\n    tensor_generator: Generator[TensorType, None, None]\n) -&gt; List[TensorType]\n</code></pre> <p>Aggregates tensors from a single-level generator into a list. This method simply collects each tensor emitted by the generator into a single list.</p> PARAMETER  DESCRIPTION <code>tensor_generator</code> <p>A generator that yields TensorType objects.</p> <p> TYPE: <code>Generator[TensorType, None, None]</code> </p> RETURNS DESCRIPTION <code>List[TensorType]</code> <p>A list containing all the tensors provided by the tensor_generator.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def __call__(\n    self, tensor_generator: Generator[TensorType, None, None]\n) -&gt; List[TensorType]:\n    \"\"\"\n    Aggregates tensors from a single-level generator into a list. This method simply\n    collects each tensor emitted by the generator into a single list.\n\n    Args:\n        tensor_generator: A generator that yields TensorType objects.\n\n    Returns:\n        A list containing all the tensors provided by the tensor_generator.\n    \"\"\"\n    return [t for t in tensor_generator]\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedSequenceAggregator","title":"NestedSequenceAggregator","text":"<p>             Bases: <code>Generic[TensorType]</code>, <code>ABC</code></p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedSequenceAggregator.__call__","title":"__call__  <code>abstractmethod</code>","text":"<pre><code>__call__(\n    nested_generators_of_tensors: Generator[\n        Generator[TensorType, None, None], None, None\n    ]\n)\n</code></pre> <p>Aggregates tensors from a generator of generators.</p> <p>Implement this method to specify how tensors, nested in two layers of generators, should be combined. Useful for complex data structures where tensors are not directly accessible in a flat list.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>@abstractmethod\ndef __call__(\n    self,\n    nested_generators_of_tensors: Generator[\n        Generator[TensorType, None, None], None, None\n    ],\n):\n    \"\"\"\n    Aggregates tensors from a generator of generators.\n\n    Implement this method to specify how tensors, nested in two layers of\n    generators, should be combined. Useful for complex data structures where tensors\n    are not directly accessible in a flat list.\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedListAggregator","title":"NestedListAggregator","text":"<p>             Bases: <code>NestedSequenceAggregator</code></p>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedListAggregator.__call__","title":"__call__","text":"<pre><code>__call__(\n    nested_generators_of_tensors: Generator[\n        Generator[TensorType, None, None], None, None\n    ]\n) -&gt; List[List[TensorType]]\n</code></pre> <p>Aggregates tensors from a nested generator structure into a list of lists.  Each inner generator is converted into a list of tensors, resulting in a nested  list structure.</p> <p>Args:      nested_generators_of_tensors: A generator of generators, where each inner         generator yields TensorType objects.</p> RETURNS DESCRIPTION <code>List[List[TensorType]]</code> <p>A list of lists, where each inner list contains tensors returned from one of the inner generators.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def __call__(\n    self,\n    nested_generators_of_tensors: Generator[\n        Generator[TensorType, None, None], None, None\n    ],\n) -&gt; List[List[TensorType]]:\n    \"\"\"\n     Aggregates tensors from a nested generator structure into a list of lists.\n     Each inner generator is converted into a list of tensors, resulting in a nested\n     list structure.\n\n     Args:\n         nested_generators_of_tensors: A generator of generators, where each inner\n            generator yields TensorType objects.\n\n    Returns:\n        A list of lists, where each inner list contains tensors returned from one\n            of the inner generators.\n    \"\"\"\n    return [list(tensor_gen) for tensor_gen in nested_generators_of_tensors]\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.LazyChunkSequence","title":"LazyChunkSequence","text":"<pre><code>LazyChunkSequence(\n    generator_factory: Callable[[], Generator[TensorType, None, None]]\n)\n</code></pre> <p>A class representing a chunked, and lazily evaluated array, where the chunking is restricted to the first dimension</p> <p>This class is designed to handle large arrays that don't fit in memory. It works by generating chunks of the array on demand and can also convert these chunks to a Zarr array for efficient storage and retrieval.</p> ATTRIBUTE DESCRIPTION <code>generator_factory</code> <p>A factory function that returns a generator. This generator yields chunks of the large array when called.</p> <p> </p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def __init__(\n    self, generator_factory: Callable[[], Generator[TensorType, None, None]]\n):\n    self.generator_factory = generator_factory\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.LazyChunkSequence.compute","title":"compute","text":"<pre><code>compute(aggregator: Optional[SequenceAggregator] = None)\n</code></pre> <p>Computes and optionally aggregates the chunks of the array using the provided aggregator. This method initiates the generation of chunks and then combines them according to the aggregator's logic.</p> PARAMETER  DESCRIPTION <code>aggregator</code> <p>An optional aggregator for combining the chunks of the array. If None, a default ListAggregator is used to simply collect the chunks into a list.</p> <p> TYPE: <code>Optional[SequenceAggregator]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <p>The aggregated result of all chunks of the array, the format of which depends on the aggregator used.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def compute(self, aggregator: Optional[SequenceAggregator] = None):\n    \"\"\"\n    Computes and optionally aggregates the chunks of the array using the provided\n    aggregator. This method initiates the generation of chunks and then\n    combines them according to the aggregator's logic.\n\n    Args:\n        aggregator: An optional aggregator for combining the chunks of\n            the array. If None, a default ListAggregator is used to simply collect\n            the chunks into a list.\n\n    Returns:\n        The aggregated result of all chunks of the array, the format of which\n            depends on the aggregator used.\n\n    \"\"\"\n    if aggregator is None:\n        aggregator = ListAggregator()\n    return aggregator(self.generator_factory())\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.LazyChunkSequence.to_zarr","title":"to_zarr","text":"<pre><code>to_zarr(\n    path_or_url: Union[str, StoreLike],\n    converter: NumpyConverter,\n    return_stored: bool = False,\n    overwrite: bool = False,\n) -&gt; Optional[Array]\n</code></pre> <p>Converts the array into Zarr format, a storage format optimized for large arrays, and stores it at the specified path or URL. This method is suitable for scenarios where the data needs to be saved for later use or for large datasets requiring efficient storage.</p> PARAMETER  DESCRIPTION <code>path_or_url</code> <p>The file path or URL where the Zarr array will be stored. Also excepts instances of zarr stores.</p> <p> TYPE: <code>Union[str, StoreLike]</code> </p> <code>converter</code> <p>A converter for transforming blocks into NumPy arrays compatible with Zarr.</p> <p> TYPE: <code>NumpyConverter</code> </p> <code>return_stored</code> <p>If True, the method returns the stored Zarr array; otherwise, it returns None.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>overwrite</code> <p>If True, overwrites existing data at the given path_or_url. If False, an error is raised in case of existing data.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Optional[Array]</code> <p>The Zarr array if return_stored is True; otherwise, None.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def to_zarr(\n    self,\n    path_or_url: Union[str, StoreLike],\n    converter: NumpyConverter,\n    return_stored: bool = False,\n    overwrite: bool = False,\n) -&gt; Optional[zarr.Array]:\n    \"\"\"\n    Converts the array into Zarr format, a storage format optimized for large\n    arrays, and stores it at the specified path or URL. This method is suitable for\n    scenarios where the data needs to be saved for later use or for large datasets\n    requiring efficient storage.\n\n    Args:\n        path_or_url: The file path or URL where the Zarr array will be stored.\n            Also excepts instances of zarr stores.\n        converter: A converter for transforming blocks into NumPy arrays\n            compatible with Zarr.\n        return_stored: If True, the method returns the stored Zarr array; otherwise,\n            it returns None.\n        overwrite: If True, overwrites existing data at the given path_or_url.\n            If False, an error is raised in case of existing data.\n\n    Returns:\n        The Zarr array if return_stored is True; otherwise, None.\n    \"\"\"\n    row_idx = 0\n    z = None\n    for block in self.generator_factory():\n        numpy_block = converter.to_numpy(block)\n\n        if z is None:\n            z = self._initialize_zarr_array(numpy_block, path_or_url, overwrite)\n\n        new_shape = self._new_shape_according_to_block(numpy_block, row_idx)\n        z.resize(new_shape)\n\n        z[row_idx : row_idx + numpy_block.shape[0]] = numpy_block\n        row_idx += numpy_block.shape[0]\n\n    return z if return_stored else None\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedLazyChunkSequence","title":"NestedLazyChunkSequence","text":"<pre><code>NestedLazyChunkSequence(\n    generator_factory: Callable[\n        [], Generator[Generator[TensorType, None, None], None, None]\n    ]\n)\n</code></pre> <p>A class representing chunked, and lazily evaluated array, where the chunking is restricted to the first two dimensions.</p> <p>This class is designed for handling large arrays where individual chunks are loaded and processed lazily. It supports converting these chunks into a Zarr array for efficient storage and retrieval, with chunking applied along the first two dimensions.</p> ATTRIBUTE DESCRIPTION <code>generator_factory</code> <p>A factory function that returns a generator of generators. Each inner generator yields chunks.</p> <p> </p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def __init__(\n    self,\n    generator_factory: Callable[\n        [], Generator[Generator[TensorType, None, None], None, None]\n    ],\n):\n    self.generator_factory = generator_factory\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedLazyChunkSequence.compute","title":"compute","text":"<pre><code>compute(aggregator: Optional[NestedSequenceAggregator] = None)\n</code></pre> <p>Computes and optionally aggregates the chunks of the array using the provided aggregator. This method initiates the generation of chunks and then combines them according to the aggregator's logic.</p> PARAMETER  DESCRIPTION <code>aggregator</code> <p>An optional aggregator for combining the chunks of the array. If None, a default NestedListAggregator is used to simply collect the chunks into a list of lists.</p> <p> TYPE: <code>Optional[NestedSequenceAggregator]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <p>The aggregated result of all chunks of the array, the format of which</p> <p>depends on the aggregator used.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def compute(self, aggregator: Optional[NestedSequenceAggregator] = None):\n    \"\"\"\n    Computes and optionally aggregates the chunks of the array using the provided\n    aggregator. This method initiates the generation of chunks and then\n    combines them according to the aggregator's logic.\n\n    Args:\n        aggregator: An optional aggregator for combining the chunks of\n            the array. If None, a default\n            [NestedListAggregator][pydvl.influence.array.NestedListAggregator]\n            is used to simply collect the chunks into a list of lists.\n\n    Returns:\n        The aggregated result of all chunks of the array, the format of which\n        depends on the aggregator used.\n\n    \"\"\"\n    if aggregator is None:\n        aggregator = NestedListAggregator()\n    return aggregator(self.generator_factory())\n</code></pre>"},{"location":"api/pydvl/influence/array/#pydvl.influence.array.NestedLazyChunkSequence.to_zarr","title":"to_zarr","text":"<pre><code>to_zarr(\n    path_or_url: Union[str, StoreLike],\n    converter: NumpyConverter,\n    return_stored: bool = False,\n    overwrite: bool = False,\n) -&gt; Optional[Array]\n</code></pre> <p>Converts the array into Zarr format, a storage format optimized for large arrays, and stores it at the specified path or URL. This method is suitable for scenarios where the data needs to be saved for later use or for large datasets requiring efficient storage.</p> PARAMETER  DESCRIPTION <code>path_or_url</code> <p>The file path or URL where the Zarr array will be stored. Also excepts instances of zarr stores.</p> <p> TYPE: <code>Union[str, StoreLike]</code> </p> <code>converter</code> <p>A converter for transforming blocks into NumPy arrays compatible with Zarr.</p> <p> TYPE: <code>NumpyConverter</code> </p> <code>return_stored</code> <p>If True, the method returns the stored Zarr array; otherwise, it returns None.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>overwrite</code> <p>If True, overwrites existing data at the given path_or_url. If False, an error is raised in case of existing data.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Optional[Array]</code> <p>The Zarr array if return_stored is True; otherwise, None.</p> Source code in <code>src/pydvl/influence/array.py</code> <pre><code>def to_zarr(\n    self,\n    path_or_url: Union[str, StoreLike],\n    converter: NumpyConverter,\n    return_stored: bool = False,\n    overwrite: bool = False,\n) -&gt; Optional[zarr.Array]:\n    \"\"\"\n    Converts the array into Zarr format, a storage format optimized for large\n    arrays, and stores it at the specified path or URL. This method is suitable for\n    scenarios where the data needs to be saved for later use or for large datasets\n    requiring efficient storage.\n\n    Args:\n        path_or_url: The file path or URL where the Zarr array will be stored.\n            Also excepts instances of zarr stores.\n        converter: A converter for transforming blocks into NumPy arrays\n            compatible with Zarr.\n        return_stored: If True, the method returns the stored Zarr array;\n            otherwise, it returns None.\n        overwrite: If True, overwrites existing data at the given path_or_url.\n            If False, an error is raised in case of existing data.\n\n    Returns:\n        The Zarr array if return_stored is True; otherwise, None.\n    \"\"\"\n\n    row_idx = 0\n    z = None\n    numpy_block = None\n    for row_blocks in self.generator_factory():\n        col_idx = 0\n        for block in row_blocks:\n            numpy_block = converter.to_numpy(block)\n            if z is None:\n                z = self._initialize_zarr_array(numpy_block, path_or_url, overwrite)\n            new_shape = self._new_shape_according_to_block(\n                z, numpy_block, row_idx, col_idx\n            )\n            z.resize(new_shape)\n            idx_slice_to_update = self._idx_slice_for_update(\n                numpy_block, row_idx, col_idx\n            )\n            z[idx_slice_to_update] = numpy_block\n\n            col_idx += numpy_block.shape[1]\n\n        if numpy_block is None:\n            raise ValueError(\"Generator is empty\")\n\n        row_idx += numpy_block.shape[0]\n\n    return z if return_stored else None\n</code></pre>"},{"location":"api/pydvl/influence/base_influence_function_model/","title":"Base influence function model","text":""},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model","title":"pydvl.influence.base_influence_function_model","text":""},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceMode","title":"InfluenceMode","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Enum representation for the types of influence.</p> ATTRIBUTE DESCRIPTION <code>Up</code> <p>Approximating the influence of a point</p> <p> </p> <code>Perturbation</code> <p>Perturbation definition of the influence score</p> <p> </p>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel","title":"InfluenceFunctionModel","text":"<p>             Bases: <code>Generic[TensorType, DataLoaderType]</code>, <code>ABC</code></p> <p>Generic abstract base class for computing influence related quantities. For a specific influence algorithm and tensor framework, inherit from this base class</p>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel.n_parameters","title":"n_parameters  <code>abstractmethod</code> <code>property</code>","text":"<pre><code>n_parameters\n</code></pre> <p>Number of trainable parameters of the underlying model</p>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel.is_thread_safe","title":"is_thread_safe  <code>abstractmethod</code> <code>property</code>","text":"<pre><code>is_thread_safe: bool\n</code></pre> <p>Whether the influence computation is thread safe</p>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel.is_fitted","title":"is_fitted  <code>abstractmethod</code> <code>property</code>","text":"<pre><code>is_fitted\n</code></pre> <p>Override this, to expose the fitting status of the instance.</p>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel.fit","title":"fit  <code>abstractmethod</code>","text":"<pre><code>fit(data: DataLoaderType) -&gt; InfluenceFunctionModel\n</code></pre> <p>Override this method to fit the influence function model to training data, e.g. pre-compute hessian matrix or matrix decompositions</p> PARAMETER  DESCRIPTION <code>data</code> <p> TYPE: <code>DataLoaderType</code> </p> RETURNS DESCRIPTION <code>InfluenceFunctionModel</code> <p>The fitted instance</p> Source code in <code>src/pydvl/influence/base_influence_function_model.py</code> <pre><code>@abstractmethod\ndef fit(self, data: DataLoaderType) -&gt; InfluenceFunctionModel:\n    \"\"\"\n    Override this method to fit the influence function model to training data,\n    e.g. pre-compute hessian matrix or matrix decompositions\n\n    Args:\n        data:\n\n    Returns:\n        The fitted instance\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/base_influence_function_model/#pydvl.influence.base_influence_function_model.InfluenceFunctionModel.influences_from_factors","title":"influences_from_factors  <code>abstractmethod</code>","text":"<pre><code>influences_from_factors(\n    z_test_factors: TensorType,\n    x: TensorType,\n    y: TensorType,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; TensorType\n</code></pre> <p>Override this method to implement the computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed array, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>TensorType</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>TensorType</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>TensorType</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/base_influence_function_model.py</code> <pre><code>@abstractmethod\ndef influences_from_factors(\n    self,\n    z_test_factors: TensorType,\n    x: TensorType,\n    y: TensorType,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; TensorType:\n    r\"\"\"\n    Override this method to implement the computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$.\n\n    Args:\n        z_test_factors: pre-computed array, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/","title":"Influence calculator","text":""},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator","title":"pydvl.influence.influence_calculator","text":"<p>This module provides functionality for calculating influences for large amount of data. The computation is based on a chunk computation model in the form of an instance of InfluenceFunctionModel, which is mapped over collection of chunks.</p>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DisableClientSingleThreadCheck","title":"DisableClientSingleThreadCheck","text":"<p>This type can be provided to the initialization of a DaskInfluenceCalculator instead of a distributed client object. It is useful in those scenarios, where the user want to disable the checking for thread-safety in the initialization phase, e.g. when using the single machine synchronous scheduler for debugging purposes.</p> Example <pre><code>from pydvl.influence import DisableClientThreadingCheck\n\nda_calc = DaskInfluenceCalculator(if_model,\n                                  TorchNumpyConverter(),\n                                  DisableClientThreadingCheck)\nda_influences = da_calc.influences(da_x_test, da_y_test, da_x, da_y)\nda_influences.compute(scheduler='synchronous')\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DaskInfluenceCalculator","title":"DaskInfluenceCalculator","text":"<pre><code>DaskInfluenceCalculator(\n    influence_function_model: InfluenceFunctionModel,\n    converter: NumpyConverter,\n    client: Union[Client, Type[DisableClientSingleThreadCheck]],\n)\n</code></pre> <p>This class is designed to compute influences over dask.array.Array collections, leveraging the capabilities of Dask for distributed computing and parallel processing. It requires an influence computation model of type InfluenceFunctionModel, which defines how influences are computed on a chunk of data. Essentially, this class functions by mapping the influence function model across the various chunks of a dask.array.Array collection.</p> PARAMETER  DESCRIPTION <code>influence_function_model</code> <p>instance of type InfluenceFunctionModel, that specifies the computation logic for influence on data chunks. It's a pivotal part of the calculator, determining how influence is computed and applied across the data array.</p> <p> TYPE: <code>InfluenceFunctionModel</code> </p> <code>converter</code> <p>A utility for converting numpy arrays to TensorType objects, facilitating the interaction between numpy arrays and the influence function model.</p> <p> TYPE: <code>NumpyConverter</code> </p> <code>client</code> <p>This parameter accepts either of two types:</p> <ol> <li> <p>A distributed Client object</p> </li> <li> <p>The special type DisableClientSingleThreadCheck, which serves as a flag to bypass certain checks.</p> </li> </ol> <p>During initialization, the system verifies if all workers are operating in single-threaded mode when the provided influence_function_model is designated as not thread-safe (indicated by the <code>is_thread_safe</code> property). If this condition is not met, the initialization will raise a specific error, signaling a potential thread-safety conflict.</p> <p>To intentionally skip this safety check (e.g., for debugging purposes using the single machine synchronous scheduler), you can supply the DisableClientSingleThreadCheck type.</p> <p> TYPE: <code>Union[Client, Type[DisableClientSingleThreadCheck]]</code> </p> <p>Warning</p> <p>Make sure to set <code>threads_per_worker=1</code>, when using the distributed scheduler for computing, if your implementation of InfluenceFunctionModel is not thread-safe. <pre><code>client = Client(threads_per_worker=1)\n</code></pre> For details on dask schedulers see the official documentation.</p> Example <pre><code>import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom pydvl.influence import DaskInfluenceCalculator\nfrom pydvl.influence.torch import CgInfluence\nfrom pydvl.influence.torch.util import (\n    torch_dataset_to_dask_array,\n    TorchNumpyConverter,\n)\nfrom distributed import Client\n\n# Possible some out of memory large Dataset\ntrain_data_set: Dataset = LargeDataSet(...)\ntest_data_set: Dataset = LargeDataSet(...)\n\ntrain_dataloader = DataLoader(train_data_set)\ninfl_model = CgInfluence(model, loss, hessian_regularization=0.01)\ninfl_model = if_model.fit(train_dataloader)\n\n# wrap your input data into dask arrays\nchunk_size = 10\nda_x, da_y = torch_dataset_to_dask_array(train_data_set, chunk_size=chunk_size)\nda_x_test, da_y_test = torch_dataset_to_dask_array(test_data_set,\n                                                   chunk_size=chunk_size)\n\n# use only one thread for scheduling, due to non-thread safety of some torch\n# operations\nclient = Client(n_workers=4, threads_per_worker=1)\n\ninfl_calc = DaskInfluenceCalculator(infl_model,\n                                    TorchNumpyConverter(device=torch.device(\"cpu\")),\n                                    client)\nda_influences = infl_calc.influences(da_x_test, da_y_test, da_x, da_y)\n# da_influences is a dask.array.Array\n\n# trigger computation and write chunks to disk in parallel\nda_influences.to_zarr(\"path/or/url\")\n</code></pre> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def __init__(\n    self,\n    influence_function_model: InfluenceFunctionModel,\n    converter: NumpyConverter,\n    client: Union[Client, Type[DisableClientSingleThreadCheck]],\n):\n    self._n_parameters = influence_function_model.n_parameters\n    self.influence_function_model = influence_function_model\n    self.numpy_converter = converter\n\n    if isinstance(client, type(DisableClientSingleThreadCheck)):\n        logger.warning(DisableClientSingleThreadCheck.warning_msg())\n        self.influence_function_model = delayed(influence_function_model)\n    elif isinstance(client, Client):\n        self._validate_client(client, influence_function_model)\n        self.influence_function_model = client.scatter(\n            influence_function_model, broadcast=True\n        )\n    else:\n        raise ValueError(\n            \"The 'client' parameter \"\n            \"must either be a distributed.Client object or the\"\n            \"type 'DisableClientSingleThreadCheck'.\"\n        )\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DaskInfluenceCalculator.n_parameters","title":"n_parameters  <code>property</code>","text":"<pre><code>n_parameters\n</code></pre> <p>Number of trainable parameters of the underlying model used in the batch computation</p>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DaskInfluenceCalculator.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Array, y: Array) -&gt; Array\n</code></pre> <p>Computes the expression</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradients are computed for the chunks of \\((x, y)\\).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Array</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Array</code> </p> RETURNS DESCRIPTION <code>Array</code> <p>dask.array.Array representing the element-wise inverse Hessian matrix vector products for the provided batch.</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influence_factors(self, x: da.Array, y: da.Array) -&gt; da.Array:\n    r\"\"\"\n    Computes the expression\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradients are computed for the chunks of $(x, y)$.\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        [dask.array.Array][dask.array.Array] representing the element-wise inverse\n            Hessian matrix vector products for the provided batch.\n\n    \"\"\"\n\n    self._validate_aligned_chunking(x, y)\n    self._validate_dimensions_not_chunked(x)\n    self._validate_dimensions_not_chunked(y)\n\n    def func(x_numpy: NDArray, y_numpy: NDArray, model: InfluenceFunctionModel):\n        factors = model.influence_factors(\n            self.numpy_converter.from_numpy(x_numpy),\n            self.numpy_converter.from_numpy(y_numpy),\n        )\n        return self.numpy_converter.to_numpy(factors)\n\n    chunks = []\n    for x_chunk, y_chunk, chunk_size in zip(\n        x.to_delayed(), y.to_delayed(), x.chunks[0]\n    ):\n        chunk_shape = (chunk_size, self.n_parameters)\n        chunk_array = da.from_delayed(\n            delayed(func)(\n                x_chunk.squeeze()[()],\n                y_chunk.squeeze()[()],\n                self.influence_function_model,\n            ),\n            dtype=x.dtype,\n            shape=chunk_shape,\n        )\n        chunks.append(chunk_array)\n\n    return da.concatenate(chunks)\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DaskInfluenceCalculator.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Array,\n    y_test: Array,\n    x: Optional[Array] = None,\n    y: Optional[Array] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Array\n</code></pre> <p>Compute approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The computation is done block-wise for the chunks of the provided dask arrays.</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Array</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Array</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Array]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Array]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Array</code> <p>dask.array.Array representing the element-wise scalar products for the provided batch.</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influences(\n    self,\n    x_test: da.Array,\n    y_test: da.Array,\n    x: Optional[da.Array] = None,\n    y: Optional[da.Array] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; da.Array:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})),\n    \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The computation is done block-wise\n    for the chunks of the provided dask arrays.\n\n    Args:\n        x_test: model input to use in the gradient computations of\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        [dask.array.Array][dask.array.Array] representing the element-wise scalar\n            products for the provided batch.\n\n    \"\"\"\n\n    self._validate_aligned_chunking(x_test, y_test)\n    self._validate_dimensions_not_chunked(x_test)\n    self._validate_dimensions_not_chunked(y_test)\n\n    if (x is None) != (y is None):\n        if x is None:\n            raise ValueError(\n                \"Providing labels y without providing model input x \"\n                \"is not supported\"\n            )\n        if y is None:\n            raise ValueError(\n                \"Providing model input x without labels y is not supported\"\n            )\n    elif x is not None:\n        self._validate_aligned_chunking(x, y)\n        self._validate_dimensions_not_chunked(x)\n        self._validate_dimensions_not_chunked(y)\n    else:\n        x, y = x_test, y_test\n\n    def func(\n        x_test_numpy: NDArray,\n        y_test_numpy: NDArray,\n        x_numpy: NDArray,\n        y_numpy: NDArray,\n        model: InfluenceFunctionModel,\n    ):\n        values = model.influences(\n            self.numpy_converter.from_numpy(x_test_numpy),\n            self.numpy_converter.from_numpy(y_test_numpy),\n            self.numpy_converter.from_numpy(x_numpy),\n            self.numpy_converter.from_numpy(y_numpy),\n            mode,\n        )\n        return self.numpy_converter.to_numpy(values)\n\n    un_chunked_x_shapes = [s[0] for s in x_test.chunks[1:]]\n    x_test_chunk_sizes = x_test.chunks[0]\n    x_chunk_sizes = x.chunks[0]\n    blocks = []\n    block_shape: Tuple[int, ...]\n\n    for x_test_chunk, y_test_chunk, test_chunk_size in zip(\n        x_test.to_delayed(), y_test.to_delayed(), x_test_chunk_sizes\n    ):\n        row = []\n        for x_chunk, y_chunk, chunk_size in zip(\n            x.to_delayed(), y.to_delayed(), x_chunk_sizes  # type:ignore\n        ):\n            if mode == InfluenceMode.Up:\n                block_shape = (test_chunk_size, chunk_size)\n            elif mode == InfluenceMode.Perturbation:\n                block_shape = (test_chunk_size, chunk_size, *un_chunked_x_shapes)\n            else:\n                raise UnsupportedInfluenceModeException(mode)\n\n            block_array = da.from_delayed(\n                delayed(func)(\n                    x_test_chunk.squeeze()[()],\n                    y_test_chunk.squeeze()[()],\n                    x_chunk.squeeze()[()],\n                    y_chunk.squeeze()[()],\n                    self.influence_function_model,\n                ),\n                shape=block_shape,\n                dtype=x_test.dtype,\n            )\n\n            if mode == InfluenceMode.Perturbation:\n                n_dims = block_array.ndim\n                new_order = tuple(range(2, n_dims)) + (0, 1)\n                block_array = block_array.transpose(new_order)\n\n            row.append(block_array)\n        blocks.append(row)\n\n    values_array = da.block(blocks)\n\n    if mode == InfluenceMode.Perturbation:\n        n_dims = values_array.ndim\n        new_order = (n_dims - 2, n_dims - 1) + tuple(range(n_dims - 2))\n        values_array = values_array.transpose(new_order)\n\n    return values_array\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.DaskInfluenceCalculator.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Array,\n    x: Array,\n    y: Array,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Array\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed array, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Array</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Array</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Array</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Array</code> <p>dask.array.Array representing the element-wise scalar product of the provided batch</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: da.Array,\n    x: da.Array,\n    y: da.Array,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; da.Array:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant\n    to be per sample of the batch $(x, y)$.\n\n    Args:\n        z_test_factors: pre-computed array, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n      [dask.array.Array][dask.array.Array] representing the element-wise scalar\n        product of the provided batch\n\n    \"\"\"\n    self._validate_aligned_chunking(x, y)\n    self._validate_dimensions_not_chunked(x)\n    self._validate_dimensions_not_chunked(y)\n    self._validate_dimensions_not_chunked(z_test_factors)\n\n    def func(\n        z_test_numpy: NDArray,\n        x_numpy: NDArray,\n        y_numpy: NDArray,\n        model: InfluenceFunctionModel,\n    ):\n        ups = model.influences_from_factors(\n            self.numpy_converter.from_numpy(z_test_numpy),\n            self.numpy_converter.from_numpy(x_numpy),\n            self.numpy_converter.from_numpy(y_numpy),\n            mode=mode,\n        )\n        return self.numpy_converter.to_numpy(ups)\n\n    un_chunked_x_shape = [s[0] for s in x.chunks[1:]]\n    x_chunk_sizes = x.chunks[0]\n    z_test_chunk_sizes = z_test_factors.chunks[0]\n    blocks = []\n    block_shape: Tuple[int, ...]\n\n    for z_test_chunk, z_test_chunk_size in zip(\n        z_test_factors.to_delayed(), z_test_chunk_sizes\n    ):\n        row = []\n        for x_chunk, y_chunk, chunk_size in zip(\n            x.to_delayed(), y.to_delayed(), x_chunk_sizes\n        ):\n            if mode == InfluenceMode.Perturbation:\n                block_shape = (z_test_chunk_size, chunk_size, *un_chunked_x_shape)\n            elif mode == InfluenceMode.Up:\n                block_shape = (z_test_chunk_size, chunk_size)\n            else:\n                raise UnsupportedInfluenceModeException(mode)\n\n            block_array = da.from_delayed(\n                delayed(func)(\n                    z_test_chunk.squeeze()[()],\n                    x_chunk.squeeze()[()],\n                    y_chunk.squeeze()[()],\n                    self.influence_function_model,\n                ),\n                shape=block_shape,\n                dtype=z_test_factors.dtype,\n            )\n\n            if mode == InfluenceMode.Perturbation:\n                n_dims = block_array.ndim\n                new_order = tuple(range(2, n_dims)) + (0, 1)\n                block_array = block_array.transpose(*new_order)\n\n            row.append(block_array)\n        blocks.append(row)\n\n    values_array = da.block(blocks)\n\n    if mode == InfluenceMode.Perturbation:\n        n_dims = values_array.ndim\n        new_order = (n_dims - 2, n_dims - 1) + tuple(range(n_dims - 2))\n        values_array = values_array.transpose(*new_order)\n\n    return values_array\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.SequentialInfluenceCalculator","title":"SequentialInfluenceCalculator","text":"<pre><code>SequentialInfluenceCalculator(influence_function_model: InfluenceFunctionModel)\n</code></pre> <p>This class serves as a simple wrapper for processing batches of data in a sequential manner. It is particularly useful in scenarios where parallel or distributed processing is not required or not feasible. The core functionality of this class is to apply a specified influence computation model, of type InfluenceFunctionModel, to batches of data one at a time.</p> PARAMETER  DESCRIPTION <code>influence_function_model</code> <p>An instance of type     [InfluenceFunctionModel]     [pydvl.influence.base_influence_function_model.InfluenceFunctionModel], that     specifies the computation logic for influence on data chunks.</p> <p> TYPE: <code>InfluenceFunctionModel</code> </p> Example <pre><code>from pydvl.influence import SequentialInfluenceCalculator\nfrom pydvl.influence.torch.util import (\nNestedTorchCatAggregator,\nTorchNumpyConverter,\n)\nfrom pydvl.influence.torch import CgInfluence\n\nbatch_size = 10\ntrain_dataloader = DataLoader(..., batch_size=batch_size)\ntest_dataloader = DataLoader(..., batch_size=batch_size)\n\ninfl_model = CgInfluence(model, loss, hessian_regularization=0.01)\ninfl_model = infl_model.fit(train_dataloader)\n\ninfl_calc = SequentialInfluenceCalculator(if_model)\n\n# this does not trigger the computation\nlazy_influences = infl_calc.influences(test_dataloader, train_dataloader)\n\n# trigger computation and pull the result into main memory, result is the full\n# tensor for all combinations of the two loaders\ninfluences = lazy_influences.compute(aggregator=NestedTorchCatAggregator())\n# or\n# trigger computation and write results chunk-wise to disk using zarr in a\n# sequential manner\nlazy_influences.to_zarr(\"local_path/or/url\", TorchNumpyConverter())\n</code></pre> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def __init__(\n    self,\n    influence_function_model: InfluenceFunctionModel,\n):\n    self.influence_function_model = influence_function_model\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.SequentialInfluenceCalculator.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(\n    data_iterable: Iterable[Tuple[TensorType, TensorType]]\n) -&gt; LazyChunkSequence\n</code></pre> <p>Compute the expression</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient are computed for the chunks \\((x, y)\\) of the data_iterable in a sequential manner.</p> PARAMETER  DESCRIPTION <code>data_iterable</code> <p>An iterable that returns tuples of tensors. Each tuple consists of a pair of tensors (x, y), representing input data and corresponding targets.</p> <p> TYPE: <code>Iterable[Tuple[TensorType, TensorType]]</code> </p> RETURNS DESCRIPTION <code>LazyChunkSequence</code> <p>A lazy data structure representing the chunks of the resulting tensor</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influence_factors(\n    self,\n    data_iterable: Iterable[Tuple[TensorType, TensorType]],\n) -&gt; LazyChunkSequence:\n    r\"\"\"\n    Compute the expression\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient are computed for the chunks $(x, y)$ of the data_iterable in\n    a sequential manner.\n\n    Args:\n        data_iterable: An iterable that returns tuples of tensors.\n            Each tuple consists of a pair of tensors (x, y), representing input data\n            and corresponding targets.\n\n    Returns:\n        A lazy data structure representing the chunks of the resulting tensor\n    \"\"\"\n    tensors_gen_factory = partial(self._influence_factors_gen, data_iterable)\n    return LazyChunkSequence(tensors_gen_factory)\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.SequentialInfluenceCalculator.influences","title":"influences","text":"<pre><code>influences(\n    test_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    train_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; NestedLazyChunkSequence\n</code></pre> <p>Compute approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The computation is done block-wise for the chunks of the provided data iterables and aggregated into a single tensor in memory.</p> PARAMETER  DESCRIPTION <code>test_data_iterable</code> <p>An iterable that returns tuples of tensors. Each tuple consists of a pair of tensors (x, y), representing input data and corresponding targets.</p> <p> TYPE: <code>Iterable[Tuple[TensorType, TensorType]]</code> </p> <code>train_data_iterable</code> <p>An iterable that returns tuples of tensors. Each tuple consists of a pair of tensors (x, y), representing input data and corresponding targets.</p> <p> TYPE: <code>Iterable[Tuple[TensorType, TensorType]]</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>NestedLazyChunkSequence</code> <p>A lazy data structure representing the chunks of the resulting tensor</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influences(\n    self,\n    test_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    train_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; NestedLazyChunkSequence:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})),\n    \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The computation is done block-wise for\n    the chunks of the provided\n    data iterables and aggregated into a single tensor in memory.\n\n    Args:\n        test_data_iterable: An iterable that returns tuples of tensors.\n            Each tuple consists of a pair of tensors (x, y), representing input data\n            and corresponding targets.\n        train_data_iterable: An iterable that returns tuples of tensors.\n            Each tuple consists of a pair of tensors (x, y), representing input data\n            and corresponding targets.\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        A lazy data structure representing the chunks of the resulting tensor\n\n    \"\"\"\n    nested_tensor_gen_factory = partial(\n        self._influences_gen,\n        test_data_iterable,\n        train_data_iterable,\n        mode,\n    )\n\n    return NestedLazyChunkSequence(nested_tensor_gen_factory)\n</code></pre>"},{"location":"api/pydvl/influence/influence_calculator/#pydvl.influence.influence_calculator.SequentialInfluenceCalculator.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Iterable[TensorType],\n    train_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; NestedLazyChunkSequence\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}}, \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))     \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}}, \\nabla_{x} \\nabla_{\\theta}     \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>Pre-computed iterable of tensors, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Iterable[TensorType]</code> </p> <code>train_data_iterable</code> <p>An iterable that returns tuples of tensors. Each tuple consists of a pair of tensors (x, y), representing input data and corresponding targets.</p> <p> TYPE: <code>Iterable[Tuple[TensorType, TensorType]]</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>NestedLazyChunkSequence</code> <p>A lazy data structure representing the chunks of the resulting tensor</p> Source code in <code>src/pydvl/influence/influence_calculator.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: Iterable[TensorType],\n    train_data_iterable: Iterable[Tuple[TensorType, TensorType]],\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; NestedLazyChunkSequence:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}}, \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\n        \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}}, \\nabla_{x} \\nabla_{\\theta}\n        \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$.\n\n    Args:\n        z_test_factors: Pre-computed iterable of tensors, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        train_data_iterable: An iterable that returns tuples of tensors.\n            Each tuple consists of a pair of tensors (x, y), representing input data\n            and corresponding targets.\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n      A lazy data structure representing the chunks of the resulting tensor\n\n    \"\"\"\n    nested_tensor_gen = partial(\n        self._influences_from_factors_gen,\n        z_test_factors,\n        train_data_iterable,\n        mode,\n    )\n    return NestedLazyChunkSequence(nested_tensor_gen)\n</code></pre>"},{"location":"api/pydvl/influence/torch/","title":"Torch","text":""},{"location":"api/pydvl/influence/torch/#pydvl.influence.torch","title":"pydvl.influence.torch","text":""},{"location":"api/pydvl/influence/torch/functional/","title":"Functional","text":""},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional","title":"pydvl.influence.torch.functional","text":"<p>This module provides methods for efficiently computing tensors related to first and second order derivatives of torch models, using functionality from torch.func. To indicate higher-order functions, i.e. functions which return functions, we use the naming convention <code>create_**_function</code>.</p> <p>In particular, the module contains functionality for</p> <ul> <li>Sample, batch-wise and empirical loss functions:<ul> <li>create_per_sample_loss_function</li> <li>create_batch_loss_function</li> <li>create_empirical_loss_function</li> </ul> </li> <li>Per sample gradient and jacobian product functions:<ul> <li>create_per_sample_gradient_function</li> <li>create_per_sample_mixed_derivative_function</li> <li>create_matrix_jacobian_product_function</li> </ul> </li> <li>Hessian, low rank approximation of Hessian and Hessian vector products:<ul> <li>hvp</li> <li>create_hvp_function</li> <li>create_batch_hvp_function</li> <li>hessian</li> <li>model_hessian_low_rank</li> </ul> </li> </ul>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.LowRankProductRepresentation","title":"LowRankProductRepresentation  <code>dataclass</code>","text":"<pre><code>LowRankProductRepresentation(eigen_vals: Tensor, projections: Tensor)\n</code></pre> <p>Representation of a low rank product of the form \\(H = V D V^T\\), where D is a diagonal matrix and V is orthogonal.</p> PARAMETER  DESCRIPTION <code>eigen_vals</code> <p>Diagonal of D.</p> <p> TYPE: <code>Tensor</code> </p> <code>projections</code> <p>The matrix V.</p> <p> TYPE: <code>Tensor</code> </p>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.LowRankProductRepresentation.to","title":"to","text":"<pre><code>to(device: device)\n</code></pre> <p>Move the representing tensors to a device</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def to(self, device: torch.device):\n    \"\"\"\n    Move the representing tensors to a device\n    \"\"\"\n    return LowRankProductRepresentation(\n        self.eigen_vals.to(device), self.projections.to(device)\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.hvp","title":"hvp","text":"<pre><code>hvp(\n    func: Callable[[Dict[str, Tensor]], Tensor],\n    params: Dict[str, Tensor],\n    vec: Dict[str, Tensor],\n    reverse_only: bool = True,\n) -&gt; Dict[str, Tensor]\n</code></pre> <p>Computes the Hessian-vector product (HVP) for a given function at the given parameters, i.e.</p> \\[\\nabla_{\\theta} \\nabla_{\\theta} f (\\theta)\\cdot v\\] <p>This function can operate in two modes, either reverse-mode autodiff only or both forward- and reverse-mode autodiff.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The scalar-valued function for which the HVP is computed.</p> <p> TYPE: <code>Callable[[Dict[str, Tensor]], Tensor]</code> </p> <code>params</code> <p>The parameters at which the HVP is computed.</p> <p> TYPE: <code>Dict[str, Tensor]</code> </p> <code>vec</code> <p>The vector with which the Hessian is multiplied.</p> <p> TYPE: <code>Dict[str, Tensor]</code> </p> <code>reverse_only</code> <p>Whether to use only reverse-mode autodiff (True, default) or both forward- and reverse-mode autodiff (False).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>The HVP of the function at the given parameters with the given vector.</p> Example <pre><code>&gt;&gt;&gt; def f(z): return torch.sum(z**2)\n&gt;&gt;&gt; u = torch.ones(10, requires_grad=True)\n&gt;&gt;&gt; v = torch.ones(10)\n&gt;&gt;&gt; hvp_vec = hvp(f, u, v)\n&gt;&gt;&gt; assert torch.allclose(hvp_vec, torch.full((10, ), 2.0))\n</code></pre> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def hvp(\n    func: Callable[[Dict[str, torch.Tensor]], torch.Tensor],\n    params: Dict[str, torch.Tensor],\n    vec: Dict[str, torch.Tensor],\n    reverse_only: bool = True,\n) -&gt; Dict[str, torch.Tensor]:\n    r\"\"\"\n    Computes the Hessian-vector product (HVP) for a given function at the given\n    parameters, i.e.\n\n    \\[\\nabla_{\\theta} \\nabla_{\\theta} f (\\theta)\\cdot v\\]\n\n    This function can operate in two modes, either reverse-mode autodiff only or both\n    forward- and reverse-mode autodiff.\n\n    Args:\n        func: The scalar-valued function for which the HVP is computed.\n        params: The parameters at which the HVP is computed.\n        vec: The vector with which the Hessian is multiplied.\n        reverse_only: Whether to use only reverse-mode autodiff\n            (True, default) or both forward- and reverse-mode autodiff (False).\n\n    Returns:\n        The HVP of the function at the given parameters with the given vector.\n\n    ??? Example\n\n        ```pycon\n        &gt;&gt;&gt; def f(z): return torch.sum(z**2)\n        &gt;&gt;&gt; u = torch.ones(10, requires_grad=True)\n        &gt;&gt;&gt; v = torch.ones(10)\n        &gt;&gt;&gt; hvp_vec = hvp(f, u, v)\n        &gt;&gt;&gt; assert torch.allclose(hvp_vec, torch.full((10, ), 2.0))\n        ```\n    \"\"\"\n\n    output: Dict[str, torch.Tensor]\n\n    if reverse_only:\n        _, vjp_fn = vjp(grad(func), params)\n        output = vjp_fn(vec)[0]\n    else:\n        output = jvp(grad(func), (params,), (vec,))[1]\n\n    return output\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_batch_hvp_function","title":"create_batch_hvp_function","text":"<pre><code>create_batch_hvp_function(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    reverse_only: bool = True,\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor, Tensor], Tensor]\n</code></pre> <p>Creates a function to compute Hessian-vector product (HVP) for a given model and loss function, where the Hessian information is computed for a provided batch.</p> <p>This function takes a PyTorch model, a loss function, and an optional boolean parameter. It returns a callable that computes the Hessian-vector product for batches of input data and a given vector. The computation can be performed in reverse mode only, based on the <code>reverse_only</code> parameter.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which the Hessian-vector product is to be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>The loss function. It should take two torch.Tensor objects as input and return a torch.Tensor.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>reverse_only</code> <p>If True, the Hessian-vector product is computed in reverse mode only.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor, Tensor], Tensor]</code> <p>A function that takes three <code>torch.Tensor</code> objects - input data (<code>x</code>), target data (<code>y</code>), and a vector (<code>vec</code>), and returns the Hessian-vector product of the loss evaluated on <code>x</code>, <code>y</code> times <code>vec</code>.</p> Example <pre><code># Assume `model` is a PyTorch model and `loss_fn` is a loss function.\nb_hvp_function = batch_hvp(model, loss_fn)\n\n# `x_batch`, `y_batch` are batches of input and target data,\n# and `vec` is a vector.\nhvp_result = b_hvp_function(x_batch, y_batch, vec)\n</code></pre> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_batch_hvp_function(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    reverse_only: bool = True,\n) -&gt; Callable[\n    [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor\n]:\n    r\"\"\"\n    Creates a function to compute Hessian-vector product (HVP) for a given model and\n    loss function, where the Hessian information is computed for a provided batch.\n\n    This function takes a PyTorch model, a loss function,\n    and an optional boolean parameter. It returns a callable\n    that computes the Hessian-vector product for batches of input data\n    and a given vector. The computation can be performed in reverse mode only,\n    based on the `reverse_only` parameter.\n\n    Args:\n        model: The PyTorch model for which the Hessian-vector product is to be computed.\n        loss: The loss function. It should take two\n            torch.Tensor objects as input and return a torch.Tensor.\n        reverse_only (bool, optional): If True, the Hessian-vector product is computed\n            in reverse mode only.\n\n    Returns:\n        A function that takes three `torch.Tensor` objects - input data (`x`),\n            target data (`y`), and a vector (`vec`),\n            and returns the Hessian-vector product of the loss\n            evaluated on `x`, `y` times `vec`.\n\n    ??? Example\n        ```python\n        # Assume `model` is a PyTorch model and `loss_fn` is a loss function.\n        b_hvp_function = batch_hvp(model, loss_fn)\n\n        # `x_batch`, `y_batch` are batches of input and target data,\n        # and `vec` is a vector.\n        hvp_result = b_hvp_function(x_batch, y_batch, vec)\n        ```\n    \"\"\"\n\n    def b_hvp(\n        params: Dict[str, torch.Tensor],\n        x: torch.Tensor,\n        y: torch.Tensor,\n        vec: torch.Tensor,\n    ):\n        return flatten_dimensions(\n            hvp(\n                lambda p: create_batch_loss_function(model, loss)(p, x, y),\n                params,\n                align_structure(params, vec),\n                reverse_only=reverse_only,\n            ).values()\n        )\n\n    return b_hvp\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_empirical_loss_function","title":"create_empirical_loss_function","text":"<pre><code>create_empirical_loss_function(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    data_loader: DataLoader,\n) -&gt; Callable[[Dict[str, Tensor]], Tensor]\n</code></pre> <p>Creates a function to compute the empirical loss of a given model on a given dataset. If we denote the model parameters with \\( \\theta \\), the resulting function approximates:</p> \\[     f(\\theta) = \\frac{1}{N}\\sum_{i=1}^N     \\operatorname{loss}(y_i, \\operatorname{model}(\\theta, x_i)) \\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\), where \\(N\\) is the number of all elements provided by the data_loader.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The model for which the loss should be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>The loss function to be used.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>data_loader</code> <p>The data loader for iterating over the dataset.</p> <p> TYPE: <code>DataLoader</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor]], Tensor]</code> <p>A function that computes the empirical loss of the model on the dataset for given model parameters.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_empirical_loss_function(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    data_loader: DataLoader,\n) -&gt; Callable[[Dict[str, torch.Tensor]], torch.Tensor]:\n    r\"\"\"\n    Creates a function to compute the empirical loss of a given model\n    on a given dataset. If we denote the model parameters with \\( \\theta \\),\n    the resulting function approximates:\n\n    \\[\n        f(\\theta) = \\frac{1}{N}\\sum_{i=1}^N\n        \\operatorname{loss}(y_i, \\operatorname{model}(\\theta, x_i))\n    \\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$\n    with model parameters $\\theta$, where $N$ is the number of all elements provided\n    by the data_loader.\n\n    Args:\n        model: The model for which the loss should be computed.\n        loss: The loss function to be used.\n        data_loader: The data loader for iterating over the dataset.\n\n    Returns:\n        A function that computes the empirical loss of the model on the dataset for\n            given model parameters.\n\n    \"\"\"\n\n    def empirical_loss(params: Dict[str, torch.Tensor]):\n        total_loss = to_model_device(torch.zeros((), requires_grad=True), model)\n        total_samples = to_model_device(torch.zeros(()), model)\n\n        for x, y in iter(data_loader):\n            output = functional_call(\n                model,\n                params,\n                (to_model_device(x, model),),\n            )\n            loss_value = loss(output, to_model_device(y, model))\n            total_loss = total_loss + loss_value * x.size(0)\n            total_samples += x.size(0)\n\n        return total_loss / total_samples\n\n    return empirical_loss\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_batch_loss_function","title":"create_batch_loss_function","text":"<pre><code>create_batch_loss_function(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor]\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]\n</code></pre> <p>Creates a function to compute the loss of a given model on a given batch of data, i.e. the function</p> \\[f(\\theta, x, y) = \\frac{1}{N} \\sum_{i=1}^N     \\operatorname{loss}(\\operatorname{model}(\\theta, x_i), y_i)\\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\), where \\(N\\) is the number of elements in the batch. Args:     model: The model for which the loss should be computed.     loss: The loss function to be used, which should be able to handle         a batch dimension</p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]</code> <p>A function that computes the loss of the model on a batch for given model parameters. The model parameter input to the function must take the form of a dict conform to model.named_parameters(), i.e. the keys must be a subset of the parameters and the corresponding tensor shapes must align. For the data input, the first dimension has to be the batch dimension.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_batch_loss_function(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n) -&gt; Callable[[Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], torch.Tensor]:\n    r\"\"\"\n    Creates a function to compute the loss of a given model on a given batch of data,\n    i.e. the function\n\n    \\[f(\\theta, x, y) = \\frac{1}{N} \\sum_{i=1}^N\n        \\operatorname{loss}(\\operatorname{model}(\\theta, x_i), y_i)\\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$\n    with model parameters $\\theta$, where $N$ is the number of elements in the batch.\n    Args:\n        model: The model for which the loss should be computed.\n        loss: The loss function to be used, which should be able to handle\n            a batch dimension\n\n    Returns:\n        A function that computes the loss of the model on a batch for given\n            model parameters. The model parameter input to the function must take\n            the form of a dict conform to model.named_parameters(), i.e. the keys\n            must be a subset of the parameters and the corresponding tensor shapes\n            must align. For the data input, the first dimension has to be the batch\n            dimension.\n    \"\"\"\n\n    def batch_loss(params: Dict[str, torch.Tensor], x: torch.Tensor, y: torch.Tensor):\n        outputs = functional_call(model, params, (to_model_device(x, model),))\n        return loss(outputs, y)\n\n    return batch_loss\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_hvp_function","title":"create_hvp_function","text":"<pre><code>create_hvp_function(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    data_loader: DataLoader,\n    precompute_grad: bool = True,\n    use_average: bool = True,\n    reverse_only: bool = True,\n    track_gradients: bool = False,\n) -&gt; Callable[[Tensor], Tensor]\n</code></pre> <p>Returns a function that calculates the approximate Hessian-vector product for a given vector. If you want to compute the exact hessian, i.e., pulling all data into memory and compute a full gradient computation, use the function hvp.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch module representing the model whose loss function's Hessian is to be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>data_loader</code> <p>A DataLoader instance that provides batches of data for calculating the Hessian-vector product. Each batch from the DataLoader is assumed to return a tuple where the first element is the model's input and the second element is the target output.</p> <p> TYPE: <code>DataLoader</code> </p> <code>precompute_grad</code> <p>If <code>True</code>, the full data gradient is precomputed and kept in memory, which can speed up the hessian vector product computation. Set this to <code>False</code>, if you can't afford to keep the full computation graph in memory.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>use_average</code> <p>If <code>True</code>, the returned function uses batch-wise computation via a batch loss function and averages the results. If <code>False</code>, the function uses backpropagation on the full empirical loss function, which is more accurate than averaging the batch hessians, but probably has a way higher memory usage.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>reverse_only</code> <p>Whether to use only reverse-mode autodiff or both forward- and reverse-mode autodiff. Ignored if <code>precompute_grad</code> is <code>True</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>track_gradients</code> <p>Whether to track gradients for the resulting tensor of the Hessian-vector products.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Callable[[Tensor], Tensor]</code> <p>A function that takes a single argument, a vector, and returns the</p> <code>Callable[[Tensor], Tensor]</code> <p>product of the Hessian of the <code>loss</code> function with respect to the</p> <code>Callable[[Tensor], Tensor]</code> <p><code>model</code>'s parameters and the input vector.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_hvp_function(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    data_loader: DataLoader,\n    precompute_grad: bool = True,\n    use_average: bool = True,\n    reverse_only: bool = True,\n    track_gradients: bool = False,\n) -&gt; Callable[[torch.Tensor], torch.Tensor]:\n    \"\"\"\n    Returns a function that calculates the approximate Hessian-vector product\n    for a given vector. If you want to compute the exact hessian,\n    i.e., pulling all data into memory and compute a full gradient computation, use\n    the function [hvp][pydvl.influence.torch.functional.hvp].\n\n    Args:\n        model: A PyTorch module representing the model whose loss function's\n            Hessian is to be computed.\n        loss: A callable that takes the model's output and target as input and\n            returns the scalar loss.\n        data_loader: A DataLoader instance that provides batches of data for\n            calculating the Hessian-vector product. Each batch from the\n            DataLoader is assumed to return a tuple where the first element is\n            the model's input and the second element is the target output.\n        precompute_grad: If `True`, the full data gradient is precomputed and\n            kept in memory, which can speed up the hessian vector product\n            computation. Set this to `False`, if you can't afford to keep the\n            full computation graph in memory.\n        use_average: If `True`, the returned function uses batch-wise\n            computation via\n            [a batch loss function][pydvl.influence.torch.functional.create_batch_loss_function]\n            and averages the results.\n            If `False`, the function uses backpropagation on the full\n            [empirical loss function]\n            [pydvl.influence.torch.functional.create_empirical_loss_function],\n            which is more accurate than averaging the batch hessians, but\n            probably has a way higher memory usage.\n        reverse_only: Whether to use only reverse-mode autodiff or\n            both forward- and reverse-mode autodiff. Ignored if\n            `precompute_grad` is `True`.\n        track_gradients: Whether to track gradients for the resulting tensor of\n            the Hessian-vector products.\n\n    Returns:\n        A function that takes a single argument, a vector, and returns the\n        product of the Hessian of the `loss` function with respect to the\n        `model`'s parameters and the input vector.\n    \"\"\"\n\n    if precompute_grad:\n        model_params = {k: p for k, p in model.named_parameters() if p.requires_grad}\n\n        if use_average:\n            model_dtype = next(p.dtype for p in model.parameters() if p.requires_grad)\n            total_grad_xy = torch.empty(0, dtype=model_dtype)\n            total_points = 0\n            grad_func = torch.func.grad(create_batch_loss_function(model, loss))\n            for x, y in iter(data_loader):\n                grad_xy = grad_func(\n                    model_params, to_model_device(x, model), to_model_device(y, model)\n                )\n                grad_xy = flatten_dimensions(grad_xy.values())\n                if total_grad_xy.nelement() == 0:\n                    total_grad_xy = torch.zeros_like(grad_xy)\n                total_grad_xy += grad_xy * len(x)\n                total_points += len(x)\n            total_grad_xy /= total_points\n        else:\n            total_grad_xy = torch.func.grad(\n                create_empirical_loss_function(model, loss, data_loader)\n            )(model_params)\n            total_grad_xy = flatten_dimensions(total_grad_xy.values())\n\n        def precomputed_grads_hvp_function(\n            precomputed_grads: torch.Tensor, vec: torch.Tensor\n        ) -&gt; torch.Tensor:\n            vec = to_model_device(vec, model)\n            if vec.ndim == 1:\n                vec = vec.unsqueeze(0)\n\n            z = (precomputed_grads * torch.autograd.Variable(vec)).sum(dim=1)\n\n            mvp = []\n            for i in range(len(z)):\n                mvp.append(\n                    flatten_dimensions(\n                        torch.autograd.grad(\n                            z[i], list(model_params.values()), retain_graph=True\n                        )\n                    )\n                )\n            result = torch.stack([arr.contiguous().view(-1) for arr in mvp])\n\n            if not track_gradients:\n                result = result.detach()\n\n            return result\n\n        return partial(precomputed_grads_hvp_function, total_grad_xy)\n\n    def hvp_function(vec: torch.Tensor) -&gt; torch.Tensor:\n        params = {\n            k: p if track_gradients else p.detach()\n            for k, p in model.named_parameters()\n            if p.requires_grad\n        }\n        v = align_structure(params, vec)\n        empirical_loss = create_empirical_loss_function(model, loss, data_loader)\n        return flatten_dimensions(\n            hvp(empirical_loss, params, v, reverse_only=reverse_only).values()\n        )\n\n    def avg_hvp_function(vec: torch.Tensor) -&gt; torch.Tensor:\n        n_batches = len(data_loader)\n        avg_hessian = to_model_device(torch.zeros_like(vec), model)\n        b_hvp = create_batch_hvp_function(model, loss, reverse_only)\n        params = {\n            k: p if track_gradients else p.detach()\n            for k, p in model.named_parameters()\n            if p.requires_grad\n        }\n        for t_x, t_y in iter(data_loader):\n            t_x, t_y = to_model_device(t_x, model), to_model_device(t_y, model)\n            avg_hessian += b_hvp(params, t_x, t_y, to_model_device(vec, model))\n\n        return avg_hessian / float(n_batches)\n\n    return avg_hvp_function if use_average else hvp_function\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.hessian","title":"hessian","text":"<pre><code>hessian(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    data_loader: DataLoader,\n    use_hessian_avg: bool = True,\n    track_gradients: bool = False,\n) -&gt; Tensor\n</code></pre> <p>Computes the Hessian matrix for a given model and loss function.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which the Hessian is computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>data_loader</code> <p>DataLoader providing batches of input data and corresponding ground truths.</p> <p> TYPE: <code>DataLoader</code> </p> <code>use_hessian_avg</code> <p>Flag to indicate whether the average Hessian across mini-batches should be computed. If False, the empirical loss across the entire dataset is used.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>track_gradients</code> <p>Whether to track gradients for the resulting tensor of the hessian vector products.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A tensor representing the Hessian matrix. The shape of the tensor will be (n_parameters, n_parameters), where n_parameters is the number of trainable parameters in the model.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def hessian(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    data_loader: DataLoader,\n    use_hessian_avg: bool = True,\n    track_gradients: bool = False,\n) -&gt; torch.Tensor:\n    \"\"\"\n    Computes the Hessian matrix for a given model and loss function.\n\n    Args:\n        model: The PyTorch model for which the Hessian is computed.\n        loss: A callable that computes the loss.\n        data_loader: DataLoader providing batches of input data and corresponding\n            ground truths.\n        use_hessian_avg: Flag to indicate whether the average Hessian across\n            mini-batches should be computed.\n            If False, the empirical loss across the entire dataset is used.\n        track_gradients: Whether to track gradients for the resulting tensor of\n            the hessian vector products.\n\n    Returns:\n        A tensor representing the Hessian matrix. The shape of the tensor will be\n            (n_parameters, n_parameters), where n_parameters is the number of trainable\n            parameters in the model.\n    \"\"\"\n\n    params = {\n        k: p if track_gradients else p.detach()\n        for k, p in model.named_parameters()\n        if p.requires_grad\n    }\n    n_parameters = sum([p.numel() for p in params.values()])\n    model_dtype = next((p.dtype for p in params.values()))\n\n    flat_params = flatten_dimensions(params.values())\n\n    if use_hessian_avg:\n        n_samples = 0\n        hessian_mat = to_model_device(\n            torch.zeros((n_parameters, n_parameters), dtype=model_dtype), model\n        )\n        blf = create_batch_loss_function(model, loss)\n\n        def flat_input_batch_loss_function(\n            p: torch.Tensor, t_x: torch.Tensor, t_y: torch.Tensor\n        ):\n            return blf(align_with_model(p, model), t_x, t_y)\n\n        for x, y in iter(data_loader):\n            n_samples += x.shape[0]\n            hessian_mat += x.shape[0] * torch.func.hessian(\n                flat_input_batch_loss_function\n            )(flat_params, to_model_device(x, model), to_model_device(y, model))\n\n        hessian_mat /= n_samples\n    else:\n\n        def flat_input_empirical_loss(p: torch.Tensor):\n            return create_empirical_loss_function(model, loss, data_loader)(\n                align_with_model(p, model)\n            )\n\n        hessian_mat = torch.func.jacrev(torch.func.jacrev(flat_input_empirical_loss))(\n            flat_params\n        )\n\n    return hessian_mat\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_per_sample_loss_function","title":"create_per_sample_loss_function","text":"<pre><code>create_per_sample_loss_function(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor]\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]\n</code></pre> <p>Generates a function to compute per-sample losses using PyTorch's vmap, i.e. the vector-valued function</p> \\[ f(\\theta, x, y)  = (\\operatorname{loss}(\\operatorname{model}(\\theta, x_1), y_1),     \\dots,     \\operatorname{loss}(\\operatorname{model}(\\theta, x_N), y_N)), \\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\), where \\(N\\) is the number of elements in the batch.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which per-sample losses will be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]</code> <p>A callable that computes the loss for each sample in the batch, given a dictionary of model inputs, the model's predictions, and the true values. The callable will return a tensor where each entry corresponds to the loss of the corresponding sample.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_per_sample_loss_function(\n    model: torch.nn.Module, loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor]\n) -&gt; Callable[[Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], torch.Tensor]:\n    r\"\"\"\n    Generates a function to compute per-sample losses using PyTorch's vmap,\n    i.e. the vector-valued function\n\n    \\[ f(\\theta, x, y)  = (\\operatorname{loss}(\\operatorname{model}(\\theta, x_1), y_1),\n        \\dots,\n        \\operatorname{loss}(\\operatorname{model}(\\theta, x_N), y_N)), \\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$ with\n    model parameters $\\theta$, where $N$ is the number of elements in the batch.\n\n    Args:\n        model: The PyTorch model for which per-sample losses will be computed.\n        loss: A callable that computes the loss.\n\n    Returns:\n        A callable that computes the loss for each sample in the batch,\n            given a dictionary of model inputs, the model's predictions,\n            and the true values. The callable will return a tensor where\n            each entry corresponds to the loss of the corresponding sample.\n    \"\"\"\n\n    def compute_loss(\n        params: Dict[str, torch.Tensor], x: torch.Tensor, y: torch.Tensor\n    ) -&gt; torch.Tensor:\n        outputs = functional_call(\n            model, params, (to_model_device(x.unsqueeze(0), model),)\n        )\n        return loss(outputs, y.unsqueeze(0))\n\n    vmap_loss: Callable[\n        [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], torch.Tensor\n    ] = torch.vmap(compute_loss, in_dims=(None, 0, 0))\n    return vmap_loss\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_per_sample_gradient_function","title":"create_per_sample_gradient_function","text":"<pre><code>create_per_sample_gradient_function(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor]\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor], Dict[str, Tensor]]\n</code></pre> <p>Generates a function to computes the per-sample gradient of the loss with respect to the model's parameters, i.e. the tensor-valued function</p> \\[ f(\\theta, x, y) = (\\nabla_{\\theta}\\operatorname{loss}     (\\operatorname{model}(\\theta, x_1), y_1), \\dots,     \\nabla_{\\theta}\\operatorname{loss}(\\operatorname{model}(\\theta, x_N), y_N) \\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\), where \\(N\\) is the number of elements in the batch.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which per-sample gradients will be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor], Dict[str, Tensor]]</code> <p>A callable that takes a dictionary of model parameters, the model's input, and the labels. It returns a dictionary with the same keys as the model's named parameters. Each entry in the returned dictionary corresponds to the gradient of the corresponding model parameter for each sample in the batch.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_per_sample_gradient_function(\n    model: torch.nn.Module, loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor]\n) -&gt; Callable[\n    [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], Dict[str, torch.Tensor]\n]:\n    r\"\"\"\n    Generates a function to computes the per-sample gradient of the loss with respect to\n    the model's parameters, i.e. the tensor-valued function\n\n    \\[ f(\\theta, x, y) = (\\nabla_{\\theta}\\operatorname{loss}\n        (\\operatorname{model}(\\theta, x_1), y_1), \\dots,\n        \\nabla_{\\theta}\\operatorname{loss}(\\operatorname{model}(\\theta, x_N), y_N) \\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$ with\n    model parameters $\\theta$, where $N$ is the number of elements in the batch.\n\n    Args:\n        model: The PyTorch model for which per-sample gradients will be computed.\n        loss: A callable that computes the loss.\n\n    Returns:\n        A callable that takes a dictionary of model parameters, the model's input,\n            and the labels. It returns a dictionary with the same keys as the model's\n            named parameters. Each entry in the returned dictionary corresponds to\n            the gradient of the corresponding model parameter for each sample\n            in the batch.\n\n    \"\"\"\n\n    per_sample_grad: Callable[\n        [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], Dict[str, torch.Tensor]\n    ] = torch.func.jacrev(create_per_sample_loss_function(model, loss))\n    return per_sample_grad\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_matrix_jacobian_product_function","title":"create_matrix_jacobian_product_function","text":"<pre><code>create_matrix_jacobian_product_function(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor], g: Tensor\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]\n</code></pre> <p>Generates a function to computes the matrix-Jacobian product (MJP) of the per-sample loss with respect to the model's parameters, i.e. the function</p> \\[ f(\\theta, x, y) = g \\, @ \\, (\\nabla_{\\theta}\\operatorname{loss}     (\\operatorname{model}(\\theta, x_i), y_i))_i^T \\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\).</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which the MJP will be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>g</code> <p>Matrix for which the product with the Jacobian will be computed. The shape of this matrix should be consistent with the shape of the jacobian.</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor], Tensor]</code> <p>A callable that takes a dictionary of model inputs, the model's input, and the labels. The callable returns the matrix-Jacobian product of the per-sample loss with respect to the model's parameters for the given matrix <code>g</code>.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_matrix_jacobian_product_function(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    g: torch.Tensor,\n) -&gt; Callable[[Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], torch.Tensor]:\n    r\"\"\"\n    Generates a function to computes the matrix-Jacobian product (MJP) of the\n    per-sample loss with respect to the model's parameters, i.e. the function\n\n    \\[ f(\\theta, x, y) = g \\, @ \\, (\\nabla_{\\theta}\\operatorname{loss}\n        (\\operatorname{model}(\\theta, x_i), y_i))_i^T \\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$ with\n    model parameters $\\theta$.\n\n    Args:\n        model: The PyTorch model for which the MJP will be computed.\n        loss: A callable that computes the loss.\n        g: Matrix for which the product with the Jacobian will be computed.\n            The shape of this matrix should be consistent with the shape of\n            the jacobian.\n\n    Returns:\n        A callable that takes a dictionary of model inputs, the model's input,\n            and the labels. The callable returns the matrix-Jacobian product of the\n            per-sample loss with respect to the model's parameters for the given\n            matrix `g`.\n\n    \"\"\"\n\n    def single_jvp(\n        params: Dict[str, torch.Tensor],\n        x: torch.Tensor,\n        y: torch.Tensor,\n        _g: torch.Tensor,\n    ):\n        return torch.func.jvp(\n            lambda p: create_per_sample_loss_function(model, loss)(p, x, y),\n            (params,),\n            (align_with_model(_g, model),),\n        )[1]\n\n    def full_jvp(params: Dict[str, torch.Tensor], x: torch.Tensor, y: torch.Tensor):\n        return torch.func.vmap(single_jvp, in_dims=(None, None, None, 0))(\n            params, x, y, g\n        )\n\n    return full_jvp\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.create_per_sample_mixed_derivative_function","title":"create_per_sample_mixed_derivative_function","text":"<pre><code>create_per_sample_mixed_derivative_function(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor]\n) -&gt; Callable[[Dict[str, Tensor], Tensor, Tensor], Dict[str, Tensor]]\n</code></pre> <p>Generates a function to computes the mixed derivatives, of the per-sample loss with respect to the model parameters and the input, i.e. the function</p> \\[ f(\\theta, x, y) = \\nabla_{\\theta}\\nabla_{x}\\operatorname{loss}     (\\operatorname{model}(\\theta, x), y) \\] <p>for a loss function \\(\\operatorname{loss}\\) and a model \\(\\operatorname{model}\\) with model parameters \\(\\theta\\).</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which the mixed derivatives are computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor], Tensor, Tensor], Dict[str, Tensor]]</code> <p>A callable that takes a dictionary of model inputs, the model's input, and the labels. The callable returns the mixed derivatives of the per-sample loss with respect to the model's parameters and input.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def create_per_sample_mixed_derivative_function(\n    model: torch.nn.Module, loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor]\n) -&gt; Callable[\n    [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], Dict[str, torch.Tensor]\n]:\n    r\"\"\"\n    Generates a function to computes the mixed derivatives, of the per-sample loss with\n    respect to the model parameters and the input, i.e. the function\n\n    \\[ f(\\theta, x, y) = \\nabla_{\\theta}\\nabla_{x}\\operatorname{loss}\n        (\\operatorname{model}(\\theta, x), y) \\]\n\n    for a loss function $\\operatorname{loss}$ and a model $\\operatorname{model}$ with\n    model parameters $\\theta$.\n\n    Args:\n        model: The PyTorch model for which the mixed derivatives are computed.\n        loss: A callable that computes the loss.\n\n    Returns:\n        A callable that takes a dictionary of model inputs, the model's input,\n            and the labels. The callable returns the mixed derivatives of the\n            per-sample loss with respect to the model's parameters and input.\n\n    \"\"\"\n\n    def compute_loss(params: Dict[str, torch.Tensor], x: torch.Tensor, y: torch.Tensor):\n        outputs = functional_call(\n            model, params, (to_model_device(x.unsqueeze(0), model),)\n        )\n        return loss(outputs, y.unsqueeze(0))\n\n    per_samp_mix_derivative: Callable[\n        [Dict[str, torch.Tensor], torch.Tensor, torch.Tensor], Dict[str, torch.Tensor]\n    ] = torch.vmap(\n        torch.func.jacrev(torch.func.grad(compute_loss, argnums=1)),\n        in_dims=(None, 0, 0),\n    )\n    return per_samp_mix_derivative\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.lanzcos_low_rank_hessian_approx","title":"lanzcos_low_rank_hessian_approx","text":"<pre><code>lanzcos_low_rank_hessian_approx(\n    hessian_vp: Callable[[Tensor], Tensor],\n    matrix_shape: Tuple[int, int],\n    hessian_perturbation: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-06,\n    max_iter: Optional[int] = None,\n    device: Optional[device] = None,\n    eigen_computation_on_gpu: bool = False,\n    torch_dtype: Optional[dtype] = None,\n) -&gt; LowRankProductRepresentation\n</code></pre> <p>Calculates a low-rank approximation of the Hessian matrix of a scalar-valued function using the implicitly restarted Lanczos algorithm, i.e.:</p> \\[ H_{\\text{approx}} = V D V^T\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.</p> PARAMETER  DESCRIPTION <code>hessian_vp</code> <p>A function that takes a vector and returns the product of the Hessian of the loss function.</p> <p> TYPE: <code>Callable[[Tensor], Tensor]</code> </p> <code>matrix_shape</code> <p>The shape of the matrix, represented by the hessian vector product.</p> <p> TYPE: <code>Tuple[int, int]</code> </p> <code>hessian_perturbation</code> <p>Regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. If not provided, it defaults to \\( \\min(\\text{model.n_parameters},     \\max(2 \\times \\text{rank_estimate} + 1, 20)) \\).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm, which stops when the difference in the approximated eigenvalue is less than <code>tol</code>. Defaults to 1e-6.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. If not provided, it defaults to \\( 10 \\cdot \\text{model.n_parameters}\\).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>device</code> <p>The device to use for executing the hessian vector product.</p> <p> TYPE: <code>Optional[device]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the provided device via cupy implementation. Ensure that either your model is small enough, or you use a small rank_estimate to fit your device's memory. If False, the eigen pair approximation is executed on the CPU with scipy's wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>torch_dtype</code> <p>If not provided, the current torch default dtype is used for conversion to torch.</p> <p> TYPE: <code>Optional[dtype]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>LowRankProductRepresentation instance that contains the top (up until rank_estimate) eigenvalues and corresponding eigenvectors of the Hessian.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def lanzcos_low_rank_hessian_approx(\n    hessian_vp: Callable[[torch.Tensor], torch.Tensor],\n    matrix_shape: Tuple[int, int],\n    hessian_perturbation: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-6,\n    max_iter: Optional[int] = None,\n    device: Optional[torch.device] = None,\n    eigen_computation_on_gpu: bool = False,\n    torch_dtype: Optional[torch.dtype] = None,\n) -&gt; LowRankProductRepresentation:\n    r\"\"\"\n    Calculates a low-rank approximation of the Hessian matrix of a scalar-valued\n    function using the implicitly restarted Lanczos algorithm, i.e.:\n\n    \\[ H_{\\text{approx}} = V D V^T\\]\n\n    where \\(D\\) is a diagonal matrix with the top (in absolute value) `rank_estimate`\n    eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.\n\n    Args:\n        hessian_vp: A function that takes a vector and returns the product of\n            the Hessian of the loss function.\n        matrix_shape: The shape of the matrix, represented by the hessian vector\n            product.\n        hessian_perturbation: Regularization parameter added to the\n            Hessian-vector product for numerical stability.\n        rank_estimate: The number of eigenvalues and corresponding eigenvectors\n            to compute. Represents the desired rank of the Hessian approximation.\n        krylov_dimension: The number of Krylov vectors to use for the Lanczos\n            method. If not provided, it defaults to\n            \\( \\min(\\text{model.n_parameters},\n                \\max(2 \\times \\text{rank_estimate} + 1, 20)) \\).\n        tol: The stopping criteria for the Lanczos algorithm, which stops when\n            the difference in the approximated eigenvalue is less than `tol`.\n            Defaults to 1e-6.\n        max_iter: The maximum number of iterations for the Lanczos method. If\n            not provided, it defaults to \\( 10 \\cdot \\text{model.n_parameters}\\).\n        device: The device to use for executing the hessian vector product.\n        eigen_computation_on_gpu: If True, tries to execute the eigen pair\n            approximation on the provided device via [cupy](https://cupy.dev/)\n            implementation. Ensure that either your model is small enough, or you\n            use a small rank_estimate to fit your device's memory. If False, the\n            eigen pair approximation is executed on the CPU with scipy's wrapper to\n            ARPACK.\n        torch_dtype: If not provided, the current torch default dtype is used for\n            conversion to torch.\n\n    Returns:\n        [LowRankProductRepresentation]\n            [pydvl.influence.torch.functional.LowRankProductRepresentation]\n            instance that contains the top (up until rank_estimate) eigenvalues\n            and corresponding eigenvectors of the Hessian.\n    \"\"\"\n\n    torch_dtype = torch.get_default_dtype() if torch_dtype is None else torch_dtype\n\n    if eigen_computation_on_gpu:\n        try:\n            import cupy as cp\n            from cupyx.scipy.sparse.linalg import LinearOperator, eigsh\n            from torch.utils.dlpack import from_dlpack, to_dlpack\n        except ImportError as e:\n            raise ImportError(\n                f\"Try to install missing dependencies or set eigen_computation_on_gpu \"\n                f\"to False: {e}\"\n            )\n\n        if device is None:\n            raise ValueError(\n                \"Without setting an explicit device, cupy is not supported\"\n            )\n\n        def to_torch_conversion_function(x: cp.NDArray) -&gt; torch.Tensor:\n            return from_dlpack(x.toDlpack()).to(torch_dtype)\n\n        def mv(x):\n            x = to_torch_conversion_function(x)\n            y = hessian_vp(x) + hessian_perturbation * x\n            return cp.from_dlpack(to_dlpack(y))\n\n    else:\n        from scipy.sparse.linalg import LinearOperator, eigsh\n\n        def mv(x):\n            x_torch = torch.as_tensor(x, device=device, dtype=torch_dtype)\n            y = (\n                (hessian_vp(x_torch) + hessian_perturbation * x_torch)\n                .detach()\n                .cpu()\n                .numpy()\n            )\n            return y\n\n        to_torch_conversion_function = partial(torch.as_tensor, dtype=torch_dtype)\n\n    try:\n        eigen_vals, eigen_vecs = eigsh(\n            LinearOperator(matrix_shape, matvec=mv),\n            k=rank_estimate,\n            maxiter=max_iter,\n            tol=tol,\n            ncv=krylov_dimension,\n            return_eigenvectors=True,\n        )\n\n    except ArpackNoConvergence as e:\n        logger.warning(\n            f\"ARPACK did not converge for parameters {max_iter=}, {tol=}, \"\n            f\"{krylov_dimension=}, {rank_estimate=}. \\n \"\n            f\"Returning the best approximation found so far. \"\n            f\"Use those with care or modify parameters.\\n Original error: {e}\"\n        )\n\n        eigen_vals, eigen_vecs = e.eigenvalues, e.eigenvectors\n\n    eigen_vals = to_torch_conversion_function(eigen_vals)\n    eigen_vecs = to_torch_conversion_function(eigen_vecs)\n\n    return LowRankProductRepresentation(eigen_vals, eigen_vecs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.model_hessian_low_rank","title":"model_hessian_low_rank","text":"<pre><code>model_hessian_low_rank(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    training_data: DataLoader,\n    hessian_perturbation: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-06,\n    max_iter: Optional[int] = None,\n    eigen_computation_on_gpu: bool = False,\n    precompute_grad: bool = False,\n) -&gt; LowRankProductRepresentation\n</code></pre> <p>Calculates a low-rank approximation of the Hessian matrix of the model's loss function using the implicitly restarted Lanczos algorithm, i.e.</p> \\[ H_{\\text{approx}} = V D V^T\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model instance. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> </p> <code>training_data</code> <p>A DataLoader instance that provides the model's training data. Used in calculating the Hessian-vector products.</p> <p> TYPE: <code>DataLoader</code> </p> <code>hessian_perturbation</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. If not provided, it defaults to min(model.n_parameters,     max(2*rank_estimate + 1, 20)).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm, which stops when the difference in the approximated eigenvalue is less than <code>tol</code>. Defaults to 1e-6.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. If not provided, it defaults to 10*model.n_parameters.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the provided device via cupy implementation. Make sure, that either your model is small enough or you use a small rank_estimate to fit your device's memory. If False, the eigen pair approximation is executed on the CPU by scipy wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>precompute_grad</code> <p>If True, the full data gradient is precomputed and kept in memory, which can speed up the hessian vector product computation. Set this to False, if you can't afford to keep the full computation graph in memory.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>LowRankProductRepresentation instance that contains the top (up until rank_estimate) eigenvalues and corresponding eigenvectors of the Hessian.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def model_hessian_low_rank(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    training_data: DataLoader,\n    hessian_perturbation: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-6,\n    max_iter: Optional[int] = None,\n    eigen_computation_on_gpu: bool = False,\n    precompute_grad: bool = False,\n) -&gt; LowRankProductRepresentation:\n    r\"\"\"\n    Calculates a low-rank approximation of the Hessian matrix of the model's\n    loss function using the implicitly restarted Lanczos algorithm, i.e.\n\n    \\[ H_{\\text{approx}} = V D V^T\\]\n\n    where \\(D\\) is a diagonal matrix with the top (in absolute value) `rank_estimate`\n    eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.\n\n\n    Args:\n        model: A PyTorch model instance. The Hessian will be calculated with respect to\n            this model's parameters.\n        loss : A callable that computes the loss.\n        training_data: A DataLoader instance that provides the model's training data.\n            Used in calculating the Hessian-vector products.\n        hessian_perturbation: Optional regularization parameter added to the\n            Hessian-vector product for numerical stability.\n        rank_estimate: The number of eigenvalues and corresponding eigenvectors to\n            compute. Represents the desired rank of the Hessian approximation.\n        krylov_dimension: The number of Krylov vectors to use for the Lanczos method.\n            If not provided, it defaults to min(model.n_parameters,\n                max(2*rank_estimate + 1, 20)).\n        tol: The stopping criteria for the Lanczos algorithm,\n            which stops when the difference in the approximated eigenvalue is less than\n            `tol`. Defaults to 1e-6.\n        max_iter: The maximum number of iterations for the Lanczos method.\n            If not provided, it defaults to 10*model.n_parameters.\n        eigen_computation_on_gpu: If True, tries to execute the eigen pair approximation\n            on the provided device via cupy implementation.\n            Make sure, that either your model is small enough or you use a\n            small rank_estimate to fit your device's memory.\n            If False, the eigen pair approximation is executed on the CPU by\n            scipy wrapper to ARPACK.\n        precompute_grad: If True, the full data gradient is precomputed and kept\n            in memory, which can speed up the hessian vector product computation.\n            Set this to False, if you can't afford to keep the full computation graph\n            in memory.\n\n    Returns:\n        [LowRankProductRepresentation]\n            [pydvl.influence.torch.functional.LowRankProductRepresentation]\n            instance that contains the top (up until rank_estimate) eigenvalues\n            and corresponding eigenvectors of the Hessian.\n    \"\"\"\n    raw_hvp = create_hvp_function(\n        model, loss, training_data, use_average=True, precompute_grad=precompute_grad\n    )\n    n_params = sum([p.numel() for p in model.parameters() if p.requires_grad])\n    device = next(model.parameters()).device\n    return lanzcos_low_rank_hessian_approx(\n        hessian_vp=raw_hvp,\n        matrix_shape=(n_params, n_params),\n        hessian_perturbation=hessian_perturbation,\n        rank_estimate=rank_estimate,\n        krylov_dimension=krylov_dimension,\n        tol=tol,\n        max_iter=max_iter,\n        device=device,\n        eigen_computation_on_gpu=eigen_computation_on_gpu,\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.randomized_nystroem_approximation","title":"randomized_nystroem_approximation","text":"<pre><code>randomized_nystroem_approximation(\n    mat_mat_prod: Union[Tensor, Callable[[Tensor], Tensor]],\n    input_dim: int,\n    rank: int,\n    input_type: dtype,\n    shift_func: Optional[Callable[[Tensor], Tensor]] = None,\n    mat_vec_device: device = torch.device(\"cpu\"),\n) -&gt; LowRankProductRepresentation\n</code></pre> <p>Given a matrix vector product function (representing a symmetric positive definite matrix \\(A\\) ), computes a random Nystr\u00f6m low rank approximation of \\(A\\) in factored form, i.e.</p> \\[ A_{\\text{nys}} = (A \\Omega)(\\Omega^T A \\Omega)^{\\dagger}(A \\Omega)^T = U \\Sigma U^T \\] <p>where \\(\\Omega\\) is a standard normal random matrix.</p> PARAMETER  DESCRIPTION <code>mat_mat_prod</code> <p>A callable representing the matrix vector product</p> <p> TYPE: <code>Union[Tensor, Callable[[Tensor], Tensor]]</code> </p> <code>input_dim</code> <p>dimension of the input for the matrix vector product</p> <p> TYPE: <code>int</code> </p> <code>input_type</code> <p>data_type of inputs</p> <p> TYPE: <code>dtype</code> </p> <code>rank</code> <p>rank of the approximation</p> <p> TYPE: <code>int</code> </p> <code>shift_func</code> <p>optional function for computing the stabilizing shift in the construction of the randomized nystroem approximation, defaults to</p> \\[ \\sqrt{\\operatorname{\\text{input_dim}}} \\cdot     \\varepsilon(\\operatorname{\\text{input_type}}) \\cdot \\|A\\Omega\\|_2,\\] <p>where \\(\\varepsilon(\\operatorname{\\text{input_type}})\\) is the value of the machine precision corresponding to the data type.</p> <p> TYPE: <code>Optional[Callable[[Tensor], Tensor]]</code> DEFAULT: <code>None</code> </p> <code>mat_vec_device</code> <p>device where the matrix vector product has to be executed</p> <p> TYPE: <code>device</code> DEFAULT: <code>device('cpu')</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>object containing, \\(U\\) and \\(\\Sigma\\)</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def randomized_nystroem_approximation(\n    mat_mat_prod: Union[torch.Tensor, Callable[[torch.Tensor], torch.Tensor]],\n    input_dim: int,\n    rank: int,\n    input_type: torch.dtype,\n    shift_func: Optional[Callable[[torch.Tensor], torch.Tensor]] = None,\n    mat_vec_device: torch.device = torch.device(\"cpu\"),\n) -&gt; LowRankProductRepresentation:\n    r\"\"\"\n    Given a matrix vector product function (representing a symmetric positive definite\n    matrix $A$ ), computes a random Nystr\u00f6m low rank approximation of\n    $A$ in factored form, i.e.\n\n    $$ A_{\\text{nys}} = (A \\Omega)(\\Omega^T A \\Omega)^{\\dagger}(A \\Omega)^T\n    = U \\Sigma U^T $$\n\n    where $\\Omega$ is a standard normal random matrix.\n\n    Args:\n        mat_mat_prod: A callable representing the matrix vector product\n        input_dim: dimension of the input for the matrix vector product\n        input_type: data_type of inputs\n        rank: rank of the approximation\n        shift_func: optional function for computing the stabilizing shift in the\n            construction of the randomized nystroem approximation, defaults to\n\n            $$ \\sqrt{\\operatorname{\\text{input_dim}}} \\cdot\n                \\varepsilon(\\operatorname{\\text{input_type}}) \\cdot \\|A\\Omega\\|_2,$$\n\n            where $\\varepsilon(\\operatorname{\\text{input_type}})$ is the value of the\n            machine precision corresponding to the data type.\n        mat_vec_device: device where the matrix vector product has to be executed\n\n    Returns:\n        object containing, $U$ and $\\Sigma$\n    \"\"\"\n\n    if shift_func is None:\n\n        def shift_func(x: torch.Tensor):\n            return (\n                torch.sqrt(torch.as_tensor(input_dim))\n                * torch.finfo(x.dtype).eps\n                * torch.linalg.norm(x)\n            )\n\n    _mat_mat_prod: Callable[[torch.Tensor], torch.Tensor]\n\n    if isinstance(mat_mat_prod, torch.Tensor):\n\n        def _mat_mat_prod(x: torch.Tensor):\n            return mat_mat_prod @ x\n\n    else:\n        _mat_mat_prod = mat_mat_prod\n\n    random_sample_matrix = torch.randn(\n        input_dim, rank, device=mat_vec_device, dtype=input_type\n    )\n    random_sample_matrix, _ = torch.linalg.qr(random_sample_matrix)\n\n    sketch_mat = _mat_mat_prod(random_sample_matrix)\n\n    shift = shift_func(sketch_mat)\n    sketch_mat += shift * random_sample_matrix\n    cholesky_mat = torch.matmul(random_sample_matrix.t(), sketch_mat)\n    try:\n        triangular_mat = torch.linalg.cholesky(cholesky_mat)\n    except _LinAlgError as e:\n        logger.warning(\n            f\"Encountered error in cholesky decomposition: {e}.\\n \"\n            f\"Increasing shift by smallest eigenvalue and re-compute\"\n        )\n        eigen_vals, eigen_vectors = torch.linalg.eigh(cholesky_mat)\n        shift += torch.abs(torch.min(eigen_vals))\n        eigen_vals += shift\n        triangular_mat = torch.linalg.cholesky(\n            torch.mm(eigen_vectors, torch.mm(torch.diag(eigen_vals), eigen_vectors.T))\n        )\n\n    svd_input = torch.linalg.solve_triangular(\n        triangular_mat.t(), sketch_mat, upper=True, left=False\n    )\n    left_singular_vecs, singular_vals, _ = torch.linalg.svd(\n        svd_input, full_matrices=False\n    )\n    singular_vals = torch.clamp(singular_vals**2 - shift, min=0)\n\n    return LowRankProductRepresentation(singular_vals, left_singular_vecs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.model_hessian_nystroem_approximation","title":"model_hessian_nystroem_approximation","text":"<pre><code>model_hessian_nystroem_approximation(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    data_loader: DataLoader,\n    rank: int,\n    shift_func: Optional[Callable[[Tensor], Tensor]] = None,\n) -&gt; LowRankProductRepresentation\n</code></pre> <p>Given a model, loss and a data_loader, computes a random Nystr\u00f6m low rank approximation of the corresponding Hessian matrix in factored form, i.e.</p> \\[ H_{\\text{nys}} = (H \\Omega)(\\Omega^T H \\Omega)^{+}(H \\Omega)^T = U \\Sigma U^T \\] PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model instance. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that computes the loss.</p> <p> </p> <code>data_loader</code> <p>A DataLoader instance that provides the model's training data. Used in calculating the Hessian-vector products.</p> <p> TYPE: <code>DataLoader</code> </p> <code>rank</code> <p>rank of the approximation</p> <p> TYPE: <code>int</code> </p> <code>shift_func</code> <p>optional function for computing the stabilizing shift in the construction of the randomized nystroem approximation, defaults to</p> \\[ \\sqrt{\\operatorname{\\text{input_dim}}} \\cdot     \\varepsilon(\\operatorname{\\text{input_type}}) \\cdot \\|A\\Omega\\|_2,\\] <p>where \\(\\varepsilon(\\operatorname{\\text{input_type}})\\) is the value of the machine precision corresponding to the data type.</p> <p> TYPE: <code>Optional[Callable[[Tensor], Tensor]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>object containing, \\(U\\) and \\(\\Sigma\\)</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def model_hessian_nystroem_approximation(\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    data_loader: DataLoader,\n    rank: int,\n    shift_func: Optional[Callable[[torch.Tensor], torch.Tensor]] = None,\n) -&gt; LowRankProductRepresentation:\n    r\"\"\"\n    Given a model, loss and a data_loader, computes a random Nystr\u00f6m low rank approximation of\n    the corresponding Hessian matrix in factored form, i.e.\n\n    $$ H_{\\text{nys}} = (H \\Omega)(\\Omega^T H \\Omega)^{+}(H \\Omega)^T\n    = U \\Sigma U^T $$\n\n    Args:\n        model: A PyTorch model instance. The Hessian will be calculated with respect to\n            this model's parameters.\n        loss : A callable that computes the loss.\n        data_loader: A DataLoader instance that provides the model's training data.\n            Used in calculating the Hessian-vector products.\n        rank: rank of the approximation\n        shift_func: optional function for computing the stabilizing shift in the\n            construction of the randomized nystroem approximation, defaults to\n\n            $$ \\sqrt{\\operatorname{\\text{input_dim}}} \\cdot\n                \\varepsilon(\\operatorname{\\text{input_type}}) \\cdot \\|A\\Omega\\|_2,$$\n\n            where $\\varepsilon(\\operatorname{\\text{input_type}})$ is the value of the\n            machine precision corresponding to the data type.\n\n    Returns:\n        object containing, $U$ and $\\Sigma$\n    \"\"\"\n\n    model_hvp = create_hvp_function(\n        model, loss, data_loader, precompute_grad=False, use_average=True\n    )\n    device = next((p.device for p in model.parameters()))\n    dtype = next((p.dtype for p in model.parameters()))\n    in_dim = sum((p.numel() for p in model.parameters() if p.requires_grad))\n\n    def model_hessian_mat_mat_prod(x: torch.Tensor):\n        return torch.func.vmap(model_hvp, in_dims=1, randomness=\"same\")(x).t()\n\n    return randomized_nystroem_approximation(\n        model_hessian_mat_mat_prod,\n        in_dim,\n        rank,\n        dtype,\n        shift_func=shift_func,\n        mat_vec_device=device,\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/","title":"Influence function model","text":""},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model","title":"pydvl.influence.torch.influence_function_model","text":"<p>This module implements several implementations of InfluenceFunctionModel utilizing PyTorch.</p>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel","title":"TorchInfluenceFunctionModel","text":"<pre><code>TorchInfluenceFunctionModel(\n    model: Module, loss: Callable[[Tensor, Tensor], Tensor]\n)\n</code></pre> <p>             Bases: <code>InfluenceFunctionModel[Tensor, DataLoader]</code>, <code>ABC</code></p> <p>Abstract base class for influence computation related to torch models</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n):\n    self.loss = loss\n    self.model = model\n    self._n_parameters = sum(\n        [p.numel() for p in model.parameters() if p.requires_grad]\n    )\n    self._model_device = next(\n        (p.device for p in model.parameters() if p.requires_grad)\n    )\n    self._model_params = {\n        k: p.detach() for k, p in self.model.named_parameters() if p.requires_grad\n    }\n    self._model_dtype = next(\n        (p.dtype for p in model.parameters() if p.requires_grad)\n    )\n    super().__init__()\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel.is_fitted","title":"is_fitted  <code>abstractmethod</code> <code>property</code>","text":"<pre><code>is_fitted\n</code></pre> <p>Override this, to expose the fitting status of the instance.</p>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel.fit","title":"fit  <code>abstractmethod</code>","text":"<pre><code>fit(data: DataLoaderType) -&gt; InfluenceFunctionModel\n</code></pre> <p>Override this method to fit the influence function model to training data, e.g. pre-compute hessian matrix or matrix decompositions</p> PARAMETER  DESCRIPTION <code>data</code> <p> TYPE: <code>DataLoaderType</code> </p> RETURNS DESCRIPTION <code>InfluenceFunctionModel</code> <p>The fitted instance</p> Source code in <code>src/pydvl/influence/base_influence_function_model.py</code> <pre><code>@abstractmethod\ndef fit(self, data: DataLoaderType) -&gt; InfluenceFunctionModel:\n    \"\"\"\n    Override this method to fit the influence function model to training data,\n    e.g. pre-compute hessian matrix or matrix decompositions\n\n    Args:\n        data:\n\n    Returns:\n        The fitted instance\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute the approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute the approximation of\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle\n    \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle\n    \\]\n\n    for the perturbation type influence case. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x_test: model input to use in the gradient computations\n            of $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    t: torch.Tensor = super().influences(x_test, y_test, x, y, mode=mode)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.TorchInfluenceFunctionModel.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.DirectInfluence","title":"DirectInfluence","text":"<pre><code>DirectInfluence(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    hessian_regularization: float = 0.0,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Given a model and training data, it finds x such that \\(Hx = b\\), with \\(H\\) being the model hessian.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns   the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>hessian_regularization</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    hessian_regularization: float = 0.0,\n):\n    super().__init__(model, loss)\n    self.hessian_regularization = hessian_regularization\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.DirectInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.DirectInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.DirectInfluence.fit","title":"fit","text":"<pre><code>fit(data: DataLoader) -&gt; DirectInfluence\n</code></pre> <p>Compute the hessian matrix based on a provided dataloader.</p> PARAMETER  DESCRIPTION <code>data</code> <p>The data to compute the Hessian with.</p> <p> TYPE: <code>DataLoader</code> </p> RETURNS DESCRIPTION <code>DirectInfluence</code> <p>The fitted instance.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def fit(self, data: DataLoader) -&gt; DirectInfluence:\n    \"\"\"\n    Compute the hessian matrix based on a provided dataloader.\n\n    Args:\n        data: The data to compute the Hessian with.\n\n    Returns:\n        The fitted instance.\n    \"\"\"\n    self.hessian = hessian(self.model, self.loss, data)\n    return self\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.DirectInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}})),     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle, \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The action of \\(H^{-1}\\) is achieved via a direct solver using torch.linalg.solve.</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A tensor representing the element-wise scalar products for the provided batch.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>@log_duration\ndef influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n        f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle, \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n        f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The action of $H^{-1}$ is achieved\n    via a direct solver using [torch.linalg.solve][torch.linalg.solve].\n\n    Args:\n        x_test: model input to use in the gradient computations of\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        A tensor representing the element-wise scalar products for the\n            provided batch.\n\n    \"\"\"\n    return super().influences(x_test, y_test, x, y, mode=mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.CgInfluence","title":"CgInfluence","text":"<pre><code>CgInfluence(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    hessian_regularization: float = 0.0,\n    x0: Optional[Tensor] = None,\n    rtol: float = 1e-07,\n    atol: float = 1e-07,\n    maxiter: Optional[int] = None,\n    progress: bool = False,\n    precompute_grad: bool = False,\n    pre_conditioner: Optional[PreConditioner] = None,\n    use_block_cg: bool = False,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Given a model and training data, it uses conjugate gradient to calculate the inverse of the Hessian Vector Product. More precisely, it finds x such that \\(Hx = b\\), with \\(H\\) being the model hessian. For more info, see Conjugate Gradient.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns   the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>hessian_regularization</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>x0</code> <p>Initial guess for hvp. If None, defaults to b.</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>rtol</code> <p>Maximum relative tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>atol</code> <p>Absolute tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>maxiter</code> <p>Maximum number of iterations. If None, defaults to 10*len(b).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>precompute_grad</code> <p>If True, the full data gradient is precomputed and kept in memory, which can speed up the hessian vector product computation. Set this to False, if you can't afford to keep the full computation graph in memory.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>pre_conditioner</code> <p>Optional pre-conditioner to improve convergence of conjugate gradient method</p> <p> TYPE: <code>Optional[PreConditioner]</code> DEFAULT: <code>None</code> </p> <code>use_block_cg</code> <p>If True, use block variant of conjugate gradient method, which solves several right hand sides simultaneously</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    hessian_regularization: float = 0.0,\n    x0: Optional[torch.Tensor] = None,\n    rtol: float = 1e-7,\n    atol: float = 1e-7,\n    maxiter: Optional[int] = None,\n    progress: bool = False,\n    precompute_grad: bool = False,\n    pre_conditioner: Optional[PreConditioner] = None,\n    use_block_cg: bool = False,\n):\n    super().__init__(model, loss)\n    self.use_block_cg = use_block_cg\n    self.pre_conditioner = pre_conditioner\n    self.precompute_grad = precompute_grad\n    self.progress = progress\n    self.maxiter = maxiter\n    self.atol = atol\n    self.rtol = rtol\n    self.x0 = x0\n    self.hessian_regularization = hessian_regularization\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.CgInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.CgInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.CgInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute an approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}})),     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle, \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of perturbation-type influence. The approximate action of \\(H^{-1}\\) is achieved via the conjugate gradient method.</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A tensor representing the element-wise scalar products for the provided batch.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>@log_duration\ndef influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute an approximation of\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n        f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle, \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n        f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of perturbation-type influence. The approximate action of\n    $H^{-1}$ is achieved via the [conjugate gradient\n    method](https://en.wikipedia.org/wiki/Conjugate_gradient_method).\n\n    Args:\n        x_test: model input to use in the gradient computations of\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        A tensor representing the element-wise scalar products for the\n            provided batch.\n\n    \"\"\"\n    return super().influences(x_test, y_test, x, y, mode=mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.LissaInfluence","title":"LissaInfluence","text":"<pre><code>LissaInfluence(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    hessian_regularization: float = 0.0,\n    maxiter: int = 1000,\n    dampen: float = 0.0,\n    scale: float = 10.0,\n    h0: Optional[Tensor] = None,\n    rtol: float = 0.0001,\n    progress: bool = False,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Uses LISSA, Linear time Stochastic Second-Order Algorithm, to iteratively approximate the inverse Hessian. More precisely, it finds x s.t. \\(Hx = b\\), with \\(H\\) being the model's second derivative wrt. the parameters. This is done with the update</p> \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\] <p>where \\(I\\) is the identity matrix, \\(d\\) is a dampening term and \\(s\\) a scaling factor that are applied to help convergence. For details, see Linear time Stochastic Second-Order Approximation (LiSSA)</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns   the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>hessian_regularization</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>maxiter</code> <p>Maximum number of iterations.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1000</code> </p> <code>dampen</code> <p>Dampening factor, defaults to 0 for no dampening.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>scale</code> <p>Scaling factor, defaults to 10.</p> <p> TYPE: <code>float</code> DEFAULT: <code>10.0</code> </p> <code>h0</code> <p>Initial guess for hvp.</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>rtol</code> <p>tolerance to use for early stopping</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0001</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    hessian_regularization: float = 0.0,\n    maxiter: int = 1000,\n    dampen: float = 0.0,\n    scale: float = 10.0,\n    h0: Optional[torch.Tensor] = None,\n    rtol: float = 1e-4,\n    progress: bool = False,\n):\n    super().__init__(model, loss)\n    self.maxiter = maxiter\n    self.hessian_regularization = hessian_regularization\n    self.progress = progress\n    self.rtol = rtol\n    self.h0 = h0\n    self.scale = scale\n    self.dampen = dampen\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.LissaInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.LissaInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute the approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute the approximation of\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle\n    \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle\n    \\]\n\n    for the perturbation type influence case. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x_test: model input to use in the gradient computations\n            of $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    t: torch.Tensor = super().influences(x_test, y_test, x, y, mode=mode)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.LissaInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.ArnoldiInfluence","title":"ArnoldiInfluence","text":"<pre><code>ArnoldiInfluence(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    hessian_regularization: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-06,\n    max_iter: Optional[int] = None,\n    eigen_computation_on_gpu: bool = False,\n    precompute_grad: bool = False,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Solves the linear system Hx = b, where H is the Hessian of the model's loss function and b is the given right-hand side vector. It employs the [implicitly restarted Arnoldi method] (https://en.wikipedia.org/wiki/Arnoldi_iteration) for computing a partial eigen decomposition, which is used fo the inversion i.e.</p> \\[x = V D^{-1} V^T b\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors. For more information, see Arnoldi.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns   the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>hessian_regularization</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. Defaults to min(model's number of parameters, max(2 times rank_estimate + 1, 20)).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm. Ignored if <code>low_rank_representation</code> is provided.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. Ignored if <code>low_rank_representation</code> is provided.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the model's device via a cupy implementation. Ensure the model size or rank_estimate is appropriate for device memory. If False, the eigen pair approximation is executed on the CPU by the scipy wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>precompute_grad</code> <p>If True, the full data gradient is precomputed and kept in memory, which can speed up the hessian vector product computation. Set this to False, if you can't afford to keep the full computation graph in memory.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    hessian_regularization: float = 0.0,\n    rank_estimate: int = 10,\n    krylov_dimension: Optional[int] = None,\n    tol: float = 1e-6,\n    max_iter: Optional[int] = None,\n    eigen_computation_on_gpu: bool = False,\n    precompute_grad: bool = False,\n):\n    super().__init__(model, loss)\n    self.hessian_regularization = hessian_regularization\n    self.rank_estimate = rank_estimate\n    self.tol = tol\n    self.max_iter = max_iter\n    self.krylov_dimension = krylov_dimension\n    self.eigen_computation_on_gpu = eigen_computation_on_gpu\n    self.precompute_grad = precompute_grad\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.ArnoldiInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.ArnoldiInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute the approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute the approximation of\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle\n    \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle\n    \\]\n\n    for the perturbation type influence case. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x_test: model input to use in the gradient computations\n            of $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    t: torch.Tensor = super().influences(x_test, y_test, x, y, mode=mode)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.ArnoldiInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.ArnoldiInfluence.fit","title":"fit","text":"<pre><code>fit(data: DataLoader) -&gt; ArnoldiInfluence\n</code></pre> <p>Fitting corresponds to the computation of the low rank decomposition</p> \\[ V D^{-1} V^T \\] <p>of the Hessian defined by the provided data loader.</p> PARAMETER  DESCRIPTION <code>data</code> <p>The data to compute the Hessian with.</p> <p> TYPE: <code>DataLoader</code> </p> RETURNS DESCRIPTION <code>ArnoldiInfluence</code> <p>The fitted instance.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def fit(self, data: DataLoader) -&gt; ArnoldiInfluence:\n    r\"\"\"\n    Fitting corresponds to the computation of the low rank decomposition\n\n    \\[ V D^{-1} V^T \\]\n\n    of the Hessian defined by the provided data loader.\n\n    Args:\n        data: The data to compute the Hessian with.\n\n    Returns:\n        The fitted instance.\n\n    \"\"\"\n    low_rank_representation = model_hessian_low_rank(\n        self.model,\n        self.loss,\n        data,\n        hessian_perturbation=0.0,  # regularization is applied, when computing values\n        rank_estimate=self.rank_estimate,\n        krylov_dimension=self.krylov_dimension,\n        tol=self.tol,\n        max_iter=self.max_iter,\n        eigen_computation_on_gpu=self.eigen_computation_on_gpu,\n        precompute_grad=self.precompute_grad,\n    )\n    self.low_rank_representation = low_rank_representation.to(self.model_device)\n    return self\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence","title":"EkfacInfluence","text":"<pre><code>EkfacInfluence(\n    model: Module,\n    update_diagonal: bool = False,\n    hessian_regularization: float = 0.0,\n    progress: bool = False,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Approximately solves the linear system Hx = b, where H is the Hessian of a model with the empirical categorical cross entropy as loss function and b is the given right-hand side vector. It employs the EK-FAC method, which is based on the kronecker factorization of the Hessian.</p> <p>Contrary to the other influence function methods, this implementation can only be used for classification tasks with a cross entropy loss function. However, it is much faster than the other methods and can be used efficiently for very large datasets and models. For more information, see Eigenvalue Corrected K-FAC.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>update_diagonal</code> <p>If True, the diagonal values in the ekfac representation are refitted from the training data after calculating the KFAC blocks. This provides a more accurate approximation of the Hessian, but it is computationally more expensive.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>hessian_regularization</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: nn.Module,\n    update_diagonal: bool = False,\n    hessian_regularization: float = 0.0,\n    progress: bool = False,\n):\n    super().__init__(model, torch.nn.functional.cross_entropy)\n    self.hessian_regularization = hessian_regularization\n    self.update_diagonal = update_diagonal\n    self.active_layers = self._parse_active_layers()\n    self.progress = progress\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute the approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute the approximation of\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle\n    \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle\n    \\]\n\n    for the perturbation type influence case. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x_test: model input to use in the gradient computations\n            of $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    t: torch.Tensor = super().influences(x_test, y_test, x, y, mode=mode)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.fit","title":"fit","text":"<pre><code>fit(data: DataLoader) -&gt; EkfacInfluence\n</code></pre> <p>Compute the KFAC blocks for each layer of the model, using the provided data. It then creates an EkfacRepresentation object that stores the KFAC blocks for each layer, their eigenvalue decomposition and diagonal values.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def fit(self, data: DataLoader) -&gt; EkfacInfluence:\n    \"\"\"\n    Compute the KFAC blocks for each layer of the model, using the provided data.\n    It then creates an EkfacRepresentation object that stores the KFAC blocks for\n    each layer, their eigenvalue decomposition and diagonal values.\n    \"\"\"\n    forward_x, grad_y = self._get_kfac_blocks(data)\n    layers_evecs_a = {}\n    layers_evect_g = {}\n    layers_diags = {}\n    for key in self.active_layers.keys():\n        evals_a, evecs_a = torch.linalg.eigh(forward_x[key])\n        evals_g, evecs_g = torch.linalg.eigh(grad_y[key])\n        layers_evecs_a[key] = evecs_a\n        layers_evect_g[key] = evecs_g\n        layers_diags[key] = torch.kron(evals_g.view(-1, 1), evals_a.view(-1, 1))\n\n    self.ekfac_representation = EkfacRepresentation(\n        self.active_layers.keys(),\n        self.active_layers.values(),\n        layers_evecs_a.values(),\n        layers_evect_g.values(),\n        layers_diags.values(),\n    )\n    if self.update_diagonal:\n        self._update_diag(data)\n    return self\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influences_by_layer","title":"influences_by_layer","text":"<pre><code>influences_by_layer(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Dict[str, Tensor]\n</code></pre> <p>Compute the influence of the data on the test data for each layer of the model.</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>A dictionary containing the influence of the data on the test data for each</p> <code>Dict[str, Tensor]</code> <p>layer of the model, with the layer name as key.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_by_layer(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Dict[str, torch.Tensor]:\n    r\"\"\"\n    Compute the influence of the data on the test data for each layer of the model.\n\n    Args:\n        x_test: model input to use in the gradient computations of\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        A dictionary containing the influence of the data on the test data for each\n        layer of the model, with the layer name as key.\n    \"\"\"\n    if not self.is_fitted:\n        raise ValueError(\n            \"Instance must be fitted before calling influence methods on it\"\n        )\n\n    if x is None:\n        if y is not None:\n            raise ValueError(\n                \"Providing labels y, without providing model input x \"\n                \"is not supported\"\n            )\n\n        return self._symmetric_values_by_layer(\n            x_test.to(self.model_device),\n            y_test.to(self.model_device),\n            mode,\n        )\n\n    if y is None:\n        raise ValueError(\n            \"Providing model input x without providing labels y is not supported\"\n        )\n\n    return self._non_symmetric_values_by_layer(\n        x_test.to(self.model_device),\n        y_test.to(self.model_device),\n        x.to(self.model_device),\n        y.to(self.model_device),\n        mode,\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influence_factors_by_layer","title":"influence_factors_by_layer","text":"<pre><code>influence_factors_by_layer(x: Tensor, y: Tensor) -&gt; Dict[str, Tensor]\n</code></pre> <p>Computes the approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>for each layer of the model separately.</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>A dictionary containing the influence factors for each layer of the model,</p> <code>Dict[str, Tensor]</code> <p>with the layer name as key.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors_by_layer(\n    self,\n    x: torch.Tensor,\n    y: torch.Tensor,\n) -&gt; Dict[str, torch.Tensor]:\n    r\"\"\"\n    Computes the approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    for each layer of the model separately.\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        A dictionary containing the influence factors for each layer of the model,\n        with the layer name as key.\n    \"\"\"\n    if not self.is_fitted:\n        raise ValueError(\n            \"Instance must be fitted before calling influence methods on it\"\n        )\n\n    return self._solve_hvp_by_layer(\n        self._loss_grad(x.to(self.model_device), y.to(self.model_device)),\n        self.ekfac_representation,\n        self.hessian_regularization,\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.influences_from_factors_by_layer","title":"influences_from_factors_by_layer","text":"<pre><code>influences_from_factors_by_layer(\n    z_test_factors: Dict[str, Tensor],\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Dict[str, Tensor]\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case for each layer of the model separately. The gradients are meant to be per sample of the batch \\((x, y)\\).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Dict[str, Tensor]</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>A dictionary containing the influence of the data on the test data</p> <code>Dict[str, Tensor]</code> <p>for each layer of the model, with the layer name as key.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors_by_layer(\n    self,\n    z_test_factors: Dict[str, torch.Tensor],\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Dict[str, torch.Tensor]:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case for each layer of the model\n    separately. The gradients are meant to be per sample of the batch $(x,\n    y)$.\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        A dictionary containing the influence of the data on the test data\n        for each layer of the model, with the layer name as key.\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        total_grad = self._loss_grad(\n            x.to(self.model_device), y.to(self.model_device)\n        )\n        start_idx = 0\n        influences = {}\n        for layer_id, layer_z_test in z_test_factors.items():\n            end_idx = start_idx + layer_z_test.shape[1]\n            influences[layer_id] = layer_z_test @ total_grad[:, start_idx:end_idx].T\n            start_idx = end_idx\n        return influences\n    elif mode == InfluenceMode.Perturbation:\n        total_mixed_grad = self._flat_loss_mixed_grad(\n            x.to(self.model_device), y.to(self.model_device)\n        )\n        start_idx = 0\n        influences = {}\n        for layer_id, layer_z_test in z_test_factors.items():\n            end_idx = start_idx + layer_z_test.shape[1]\n            influences[layer_id] = torch.einsum(\n                \"ia,j...a-&gt;ij...\",\n                layer_z_test,\n                total_mixed_grad[:, start_idx:end_idx],\n            )\n            start_idx = end_idx\n        return influences\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.EkfacInfluence.explore_hessian_regularization","title":"explore_hessian_regularization","text":"<pre><code>explore_hessian_regularization(\n    x: Tensor, y: Tensor, regularization_values: List[float]\n) -&gt; Dict[float, Dict[str, Tensor]]\n</code></pre> <p>Efficiently computes the influence for input x and label y for each layer of the model, for different values of the hessian regularization parameter. This is done by computing the gradient of the loss function for the input x and label y only once and then solving the Hessian Vector Product for each regularization value. This is useful for finding the optimal regularization value and for exploring how robust the influence values are to changes in the regularization value.</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>regularization_values</code> <p>list of regularization values to use</p> <p> TYPE: <code>List[float]</code> </p> RETURNS DESCRIPTION <code>Dict[float, Dict[str, Tensor]]</code> <p>A dictionary containing with keys being the regularization values and values</p> <code>Dict[float, Dict[str, Tensor]]</code> <p>being dictionaries containing the influences for each layer of the model,</p> <code>Dict[float, Dict[str, Tensor]]</code> <p>with the layer name as key.</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def explore_hessian_regularization(\n    self,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    regularization_values: List[float],\n) -&gt; Dict[float, Dict[str, torch.Tensor]]:\n    \"\"\"\n    Efficiently computes the influence for input x and label y for each layer of the\n    model, for different values of the hessian regularization parameter. This is done\n    by computing the gradient of the loss function for the input x and label y only once\n    and then solving the Hessian Vector Product for each regularization value. This is\n    useful for finding the optimal regularization value and for exploring\n    how robust the influence values are to changes in the regularization value.\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n        regularization_values: list of regularization values to use\n\n    Returns:\n        A dictionary containing with keys being the regularization values and values\n        being dictionaries containing the influences for each layer of the model,\n        with the layer name as key.\n    \"\"\"\n    grad = self._loss_grad(x, y)\n    influences_by_reg_value = {}\n    for reg_value in regularization_values:\n        reg_factors = self._solve_hvp_by_layer(\n            grad, self.ekfac_representation, reg_value\n        )\n        values = {}\n        start_idx = 0\n        for layer_id, layer_fac in reg_factors.items():\n            end_idx = start_idx + layer_fac.shape[1]\n            values[layer_id] = layer_fac @ grad[:, start_idx:end_idx].T\n            start_idx = end_idx\n        influences_by_reg_value[reg_value] = values\n    return influences_by_reg_value\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.NystroemSketchInfluence","title":"NystroemSketchInfluence","text":"<pre><code>NystroemSketchInfluence(\n    model: Module,\n    loss: Callable[[Tensor, Tensor], Tensor],\n    hessian_regularization: float,\n    rank: int,\n)\n</code></pre> <p>             Bases: <code>TorchInfluenceFunctionModel</code></p> <p>Given a model and training data, it uses a low-rank approximation of the Hessian (derived via random projection Nystr\u00f6m approximation) in combination with the Sherman\u2013Morrison\u2013Woodbury formula to calculate the inverse of the Hessian Vector Product. More concrete, it computes a low-rank approximation</p> \\[\\begin{align*}     H_{\\text{nys}} &amp;= (H\\Omega)(\\Omega^TH\\Omega)^{+}(H\\Omega)^T \\\\\\                    &amp;= U \\Lambda U^T \\end{align*}\\] <p>in factorized form and approximates the action of the inverse Hessian via</p> \\[ (H_{\\text{nys}} + \\lambda I)^{-1} = U(\\Lambda+\\lambda I)U^T +     \\frac{1}{\\lambda}(I\u2212UU^T). \\] PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns   the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>hessian_regularization</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> </p> <code>rank</code> <p>rank of the low-rank approximation</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def __init__(\n    self,\n    model: torch.nn.Module,\n    loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n    hessian_regularization: float,\n    rank: int,\n):\n    super().__init__(model, loss)\n    self.hessian_regularization = hessian_regularization\n    self.rank = rank\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.NystroemSketchInfluence.influence_factors","title":"influence_factors","text":"<pre><code>influence_factors(x: Tensor, y: Tensor) -&gt; Tensor\n</code></pre> <p>Compute approximation of</p> \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\] <p>where the gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x</code> <p>model input to use in the gradient computations</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise inverse Hessian matrix vector products</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influence_factors(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute approximation of\n\n    \\[ H^{-1}\\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\]\n\n    where the gradient is meant to be per sample of the batch $(x, y)$.\n    For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x: model input to use in the gradient computations\n        y: label tensor to compute gradients\n\n    Returns:\n        Tensor representing the element-wise inverse Hessian matrix vector products\n\n    \"\"\"\n    return super().influence_factors(x, y)\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.NystroemSketchInfluence.influences","title":"influences","text":"<pre><code>influences(\n    x_test: Tensor,\n    y_test: Tensor,\n    x: Optional[Tensor] = None,\n    y: Optional[Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Compute the approximation of</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>x_test</code> <p>model input to use in the gradient computations of \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},     f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y_test</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>optional model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), if None, use \\(x=x_{\\text{test}}\\)</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>y</code> <p>optional label tensor to compute gradients</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences(\n    self,\n    x_test: torch.Tensor,\n    y_test: torch.Tensor,\n    x: Optional[torch.Tensor] = None,\n    y: Optional[torch.Tensor] = None,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Compute the approximation of\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n    f_{\\theta}(x_{\\text{test}})), \\nabla_{\\theta} \\ell(y, f_{\\theta}(x))\\rangle\n    \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[\n    \\langle H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}})),\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle\n    \\]\n\n    for the perturbation type influence case. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        x_test: model input to use in the gradient computations\n            of $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n                f_{\\theta}(x_{\\text{test}}))$\n        y_test: label tensor to compute gradients\n        x: optional model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            if None, use $x=x_{\\text{test}}$\n        y: optional label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    t: torch.Tensor = super().influences(x_test, y_test, x, y, mode=mode)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/influence_function_model/#pydvl.influence.torch.influence_function_model.NystroemSketchInfluence.influences_from_factors","title":"influences_from_factors","text":"<pre><code>influences_from_factors(\n    z_test_factors: Tensor,\n    x: Tensor,\n    y: Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; Tensor\n</code></pre> <p>Computation of</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the case of up-weighting influence, resp.</p> \\[ \\langle z_{\\text{test_factors}},     \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\] <p>for the perturbation type influence case. The gradient is meant to be per sample of the batch \\((x, y)\\). For all input tensors it is assumed, that the first dimension is the batch dimension (in case, you want to provide a single sample z, call z.unsqueeze(0) if no batch dimension is present).</p> PARAMETER  DESCRIPTION <code>z_test_factors</code> <p>pre-computed tensor, approximating \\(H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}}, f_{\\theta}(x_{\\text{test}}))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>x</code> <p>model input to use in the gradient computations \\(\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\), resp. \\(\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))\\)</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>label tensor to compute gradients</p> <p> TYPE: <code>Tensor</code> </p> <code>mode</code> <p>enum value of InfluenceMode</p> <p> TYPE: <code>InfluenceMode</code> DEFAULT: <code>Up</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>Tensor representing the element-wise scalar products for the provided batch</p> Source code in <code>src/pydvl/influence/torch/influence_function_model.py</code> <pre><code>def influences_from_factors(\n    self,\n    z_test_factors: torch.Tensor,\n    x: torch.Tensor,\n    y: torch.Tensor,\n    mode: InfluenceMode = InfluenceMode.Up,\n) -&gt; torch.Tensor:\n    r\"\"\"\n    Computation of\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the case of up-weighting influence, resp.\n\n    \\[ \\langle z_{\\text{test_factors}},\n        \\nabla_{x} \\nabla_{\\theta} \\ell(y, f_{\\theta}(x)) \\rangle \\]\n\n    for the perturbation type influence case. The gradient is meant to be per sample\n    of the batch $(x, y)$. For all input tensors it is assumed,\n    that the first dimension is the batch dimension (in case, you want to provide\n    a single sample z, call z.unsqueeze(0) if no batch dimension is present).\n\n    Args:\n        z_test_factors: pre-computed tensor, approximating\n            $H^{-1}\\nabla_{\\theta} \\ell(y_{\\text{test}},\n            f_{\\theta}(x_{\\text{test}}))$\n        x: model input to use in the gradient computations\n            $\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$,\n            resp. $\\nabla_{x}\\nabla_{\\theta}\\ell(y, f_{\\theta}(x))$\n        y: label tensor to compute gradients\n        mode: enum value of [InfluenceMode]\n            [pydvl.influence.base_influence_function_model.InfluenceMode]\n\n    Returns:\n        Tensor representing the element-wise scalar products for the provided batch\n\n    \"\"\"\n    if mode == InfluenceMode.Up:\n        return (\n            z_test_factors\n            @ self._loss_grad(x.to(self.model_device), y.to(self.model_device)).T\n        )\n    elif mode == InfluenceMode.Perturbation:\n        return torch.einsum(\n            \"ia,j...a-&gt;ij...\",\n            z_test_factors,\n            self._flat_loss_mixed_grad(\n                x.to(self.model_device), y.to(self.model_device)\n            ),\n        )\n    else:\n        raise UnsupportedInfluenceModeException(mode)\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/","title":"Pre conditioner","text":""},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner","title":"pydvl.influence.torch.pre_conditioner","text":""},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.PreConditioner","title":"PreConditioner","text":"<p>             Bases: <code>ABC</code></p> <p>Abstract base class for implementing pre-conditioners for improving the convergence of CG for systems of the form</p> \\[ ( A + \\lambda \\operatorname{I})x = \\operatorname{rhs} \\] <p>i.e. a matrix \\(M\\) such that \\(M^{-1}(A + \\lambda \\operatorname{I})\\) has a better condition number than \\(A + \\lambda \\operatorname{I}\\).</p>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.PreConditioner.fit","title":"fit  <code>abstractmethod</code>","text":"<pre><code>fit(\n    mat_mat_prod: Callable[[Tensor], Tensor],\n    size: int,\n    dtype: dtype,\n    device: device,\n    regularization: float = 0.0,\n)\n</code></pre> <p>Implement this to fit the pre-conditioner to the matrix represented by the mat_mat_prod Args:     mat_mat_prod: a callable that computes the matrix-matrix product     size: size of the matrix represented by <code>mat_mat_prod</code>     dtype: data type of the matrix represented by <code>mat_mat_prod</code>     device: device of the matrix represented by <code>mat_mat_prod</code>     regularization: regularization parameter \\(\\lambda\\) in the equation         $ ( A + \\lambda \\operatorname{I})x = \\operatorname{rhs} $ Returns:     self</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>@abstractmethod\ndef fit(\n    self,\n    mat_mat_prod: Callable[[torch.Tensor], torch.Tensor],\n    size: int,\n    dtype: torch.dtype,\n    device: torch.device,\n    regularization: float = 0.0,\n):\n    r\"\"\"\n    Implement this to fit the pre-conditioner to the matrix represented by the\n    mat_mat_prod\n    Args:\n        mat_mat_prod: a callable that computes the matrix-matrix product\n        size: size of the matrix represented by `mat_mat_prod`\n        dtype: data type of the matrix represented by `mat_mat_prod`\n        device: device of the matrix represented by `mat_mat_prod`\n        regularization: regularization parameter $\\lambda$ in the equation\n            $ ( A + \\lambda \\operatorname{I})x = \\operatorname{rhs} $\n    Returns:\n        self\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.PreConditioner.solve","title":"solve","text":"<pre><code>solve(rhs: Tensor)\n</code></pre> <p>Solve the equation \\(M@Z = \\operatorname{rhs}\\) Args:     rhs: right hand side of the equation, corresponds to the residuum vector         (or matrix) in the conjugate gradient method</p> RETURNS DESCRIPTION <p>solution \\(M^{-1}\\operatorname{rhs}\\)</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def solve(self, rhs: torch.Tensor):\n    r\"\"\"\n    Solve the equation $M@Z = \\operatorname{rhs}$\n    Args:\n        rhs: right hand side of the equation, corresponds to the residuum vector\n            (or matrix) in the conjugate gradient method\n\n    Returns:\n        solution $M^{-1}\\operatorname{rhs}$\n\n    \"\"\"\n    if not self.is_fitted:\n        raise NotFittedException(type(self))\n\n    return self._solve(rhs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.JacobiPreConditioner","title":"JacobiPreConditioner","text":"<pre><code>JacobiPreConditioner(num_samples_estimator: int = 1)\n</code></pre> <p>             Bases: <code>PreConditioner</code></p> <p>Pre-conditioner for improving the convergence of CG for systems of the form</p> \\[ ( A + \\lambda \\operatorname{I})x = \\operatorname{rhs} \\] <p>The JacobiPreConditioner uses the diagonal information of the matrix \\(A\\). The diagonal elements are not computed directly but estimated via Hutchinson's estimator.</p> \\[ M = \\frac{1}{m} \\sum_{i=1}^m u_i \\odot Au_i + \\lambda \\operatorname{I} \\] <p>where \\(u_i\\) are i.i.d. Gaussian random vectors. Works well in the case the matrix \\(A + \\lambda \\operatorname{I}\\) is diagonal dominant. For more information, see the documentation of Conjugate Gradient Args:     num_samples_estimator: number of samples to use in computation of         Hutchinson's estimator</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def __init__(self, num_samples_estimator: int = 1):\n    self.num_samples_estimator = num_samples_estimator\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.JacobiPreConditioner.solve","title":"solve","text":"<pre><code>solve(rhs: Tensor)\n</code></pre> <p>Solve the equation \\(M@Z = \\operatorname{rhs}\\) Args:     rhs: right hand side of the equation, corresponds to the residuum vector         (or matrix) in the conjugate gradient method</p> RETURNS DESCRIPTION <p>solution \\(M^{-1}\\operatorname{rhs}\\)</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def solve(self, rhs: torch.Tensor):\n    r\"\"\"\n    Solve the equation $M@Z = \\operatorname{rhs}$\n    Args:\n        rhs: right hand side of the equation, corresponds to the residuum vector\n            (or matrix) in the conjugate gradient method\n\n    Returns:\n        solution $M^{-1}\\operatorname{rhs}$\n\n    \"\"\"\n    if not self.is_fitted:\n        raise NotFittedException(type(self))\n\n    return self._solve(rhs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.JacobiPreConditioner.fit","title":"fit","text":"<pre><code>fit(\n    mat_mat_prod: Callable[[Tensor], Tensor],\n    size: int,\n    dtype: dtype,\n    device: device,\n    regularization: float = 0.0,\n)\n</code></pre> <p>Fits by computing an estimate of the diagonal of the matrix represented by <code>mat_mat_prod</code> via Hutchinson's estimator</p> PARAMETER  DESCRIPTION <code>mat_mat_prod</code> <p>a callable representing the matrix-matrix product</p> <p> TYPE: <code>Callable[[Tensor], Tensor]</code> </p> <code>size</code> <p>size of the square matrix</p> <p> TYPE: <code>int</code> </p> <code>dtype</code> <p>needed data type of inputs for the mat_mat_prod</p> <p> TYPE: <code>dtype</code> </p> <code>device</code> <p>needed device for inputs of mat_mat_prod</p> <p> TYPE: <code>device</code> </p> <code>regularization</code> <p>regularization parameter \\(\\lambda\\) in \\((A+\\lambda I)x=b\\)</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def fit(\n    self,\n    mat_mat_prod: Callable[[torch.Tensor], torch.Tensor],\n    size: int,\n    dtype: torch.dtype,\n    device: torch.device,\n    regularization: float = 0.0,\n):\n    r\"\"\"\n    Fits by computing an estimate of the diagonal of the matrix represented by\n    `mat_mat_prod` via Hutchinson's estimator\n\n    Args:\n        mat_mat_prod: a callable representing the matrix-matrix product\n        size: size of the square matrix\n        dtype: needed data type of inputs for the mat_mat_prod\n        device: needed device for inputs of mat_mat_prod\n        regularization: regularization parameter\n            $\\lambda$ in $(A+\\lambda I)x=b$\n    \"\"\"\n    random_samples = torch.randn(\n        size, self.num_samples_estimator, device=device, dtype=dtype\n    )\n    diagonal_estimate = torch.sum(\n        torch.mul(random_samples, mat_mat_prod(random_samples)), dim=1\n    )\n    diagonal_estimate /= self.num_samples_estimator\n    self._diag = diagonal_estimate\n    self._reg = regularization\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.NystroemPreConditioner","title":"NystroemPreConditioner","text":"<pre><code>NystroemPreConditioner(rank: int)\n</code></pre> <p>             Bases: <code>PreConditioner</code></p> <p>Pre-conditioner for improving the convergence of CG for systems of the form</p> \\[ (A + \\lambda \\operatorname{I})x = \\operatorname{rhs} \\] <p>The NystroemPreConditioner computes a low-rank approximation</p> \\[ A_{\\text{nys}} = (A \\Omega)(\\Omega^T A \\Omega)^{\\dagger}(A \\Omega)^T = U \\Sigma U^T, \\] <p>where \\((\\cdot)^{\\dagger}\\) denotes the Moore-Penrose inverse, and uses the matrix</p> \\[ M^{-1} = (\\lambda + \\sigma_{\\text{rank}})U(\\Sigma+     \\lambda \\operatorname{I})^{-1}U^T+(\\operatorname{I} - UU^T) \\] <p>for pre-conditioning, where \\(  \\sigma_{\\text{rank}} \\) is the smallest eigenvalue of the low-rank approximation.</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def __init__(self, rank: int):\n    self._rank = rank\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.NystroemPreConditioner.solve","title":"solve","text":"<pre><code>solve(rhs: Tensor)\n</code></pre> <p>Solve the equation \\(M@Z = \\operatorname{rhs}\\) Args:     rhs: right hand side of the equation, corresponds to the residuum vector         (or matrix) in the conjugate gradient method</p> RETURNS DESCRIPTION <p>solution \\(M^{-1}\\operatorname{rhs}\\)</p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def solve(self, rhs: torch.Tensor):\n    r\"\"\"\n    Solve the equation $M@Z = \\operatorname{rhs}$\n    Args:\n        rhs: right hand side of the equation, corresponds to the residuum vector\n            (or matrix) in the conjugate gradient method\n\n    Returns:\n        solution $M^{-1}\\operatorname{rhs}$\n\n    \"\"\"\n    if not self.is_fitted:\n        raise NotFittedException(type(self))\n\n    return self._solve(rhs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/pre_conditioner/#pydvl.influence.torch.pre_conditioner.NystroemPreConditioner.fit","title":"fit","text":"<pre><code>fit(\n    mat_mat_prod: Callable[[Tensor], Tensor],\n    size: int,\n    dtype: dtype,\n    device: device,\n    regularization: float = 0.0,\n)\n</code></pre> <p>Fits by computing a low-rank approximation of the matrix represented by <code>mat_mat_prod</code> via Nystroem approximation</p> PARAMETER  DESCRIPTION <code>mat_mat_prod</code> <p>a callable representing the matrix-matrix product</p> <p> TYPE: <code>Callable[[Tensor], Tensor]</code> </p> <code>size</code> <p>size of the square matrix</p> <p> TYPE: <code>int</code> </p> <code>dtype</code> <p>needed data type of inputs for the mat_mat_prod</p> <p> TYPE: <code>dtype</code> </p> <code>device</code> <p>needed device for inputs of mat_mat_prod</p> <p> TYPE: <code>device</code> </p> <code>regularization</code> <p>regularization parameter \\(\\lambda\\)  in \\((A+\\lambda I)x=b\\)</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> Source code in <code>src/pydvl/influence/torch/pre_conditioner.py</code> <pre><code>def fit(\n    self,\n    mat_mat_prod: Callable[[torch.Tensor], torch.Tensor],\n    size: int,\n    dtype: torch.dtype,\n    device: torch.device,\n    regularization: float = 0.0,\n):\n    r\"\"\"\n    Fits by computing a low-rank approximation of the matrix represented by\n    `mat_mat_prod` via Nystroem approximation\n\n    Args:\n        mat_mat_prod: a callable representing the matrix-matrix product\n        size: size of the square matrix\n        dtype: needed data type of inputs for the mat_mat_prod\n        device: needed device for inputs of mat_mat_prod\n        regularization: regularization parameter\n            $\\lambda$  in $(A+\\lambda I)x=b$\n    \"\"\"\n\n    self._low_rank_approx = randomized_nystroem_approximation(\n        mat_mat_prod, size, self._rank, dtype, mat_vec_device=device\n    )\n    self._regularization = regularization\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/","title":"Util","text":""},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util","title":"pydvl.influence.torch.util","text":""},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchTensorContainerType","title":"TorchTensorContainerType  <code>module-attribute</code>","text":"<pre><code>TorchTensorContainerType = Union[\n    Tensor, Collection[Tensor], Mapping[str, Tensor]\n]\n</code></pre> <p>Type for a PyTorch tensor or a container thereof.</p>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchNumpyConverter","title":"TorchNumpyConverter","text":"<pre><code>TorchNumpyConverter(device: Optional[device] = None)\n</code></pre> <p>             Bases: <code>NumpyConverter[Tensor]</code></p> <p>Helper class for converting between torch.Tensor and numpy.ndarray</p> PARAMETER  DESCRIPTION <code>device</code> <p>Optional device parameter to move the resulting torch tensors to the specified device</p> <p> TYPE: <code>Optional[device]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def __init__(self, device: Optional[torch.device] = None):\n    self.device = device\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchNumpyConverter.to_numpy","title":"to_numpy","text":"<pre><code>to_numpy(x: Tensor) -&gt; NDArray\n</code></pre> <p>Convert a detached torch.Tensor to numpy.ndarray</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def to_numpy(self, x: torch.Tensor) -&gt; NDArray:\n    \"\"\"\n    Convert a detached [torch.Tensor][torch.Tensor] to\n    [numpy.ndarray][numpy.ndarray]\n    \"\"\"\n    arr: NDArray = x.cpu().numpy()\n    return arr\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchNumpyConverter.from_numpy","title":"from_numpy","text":"<pre><code>from_numpy(x: NDArray) -&gt; Tensor\n</code></pre> <p>Convert a numpy.ndarray to torch.Tensor and optionally move it to a provided device</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def from_numpy(self, x: NDArray) -&gt; torch.Tensor:\n    \"\"\"\n    Convert a [numpy.ndarray][numpy.ndarray] to [torch.Tensor][torch.Tensor] and\n    optionally move it to a provided device\n    \"\"\"\n    t = torch.from_numpy(x)\n    if self.device is not None:\n        t = t.to(self.device)\n    return t\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchCatAggregator","title":"TorchCatAggregator","text":"<p>             Bases: <code>SequenceAggregator[Tensor]</code></p> <p>An aggregator that concatenates tensors using PyTorch's torch.cat function. Concatenation is done along the first dimension of the chunks.</p>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchCatAggregator.__call__","title":"__call__","text":"<pre><code>__call__(tensor_generator: Generator[Tensor, None, None])\n</code></pre> <p>Aggregates tensors from a single-level generator into a single tensor by concatenating them. This method is a straightforward way to combine a sequence of tensors into one larger tensor.</p> PARAMETER  DESCRIPTION <code>tensor_generator</code> <p>A generator that yields <code>torch.Tensor</code> objects.</p> <p> TYPE: <code>Generator[Tensor, None, None]</code> </p> RETURNS DESCRIPTION <p>A single tensor formed by concatenating all tensors from the generator. The concatenation is performed along the default dimension (0).</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def __call__(self, tensor_generator: Generator[torch.Tensor, None, None]):\n    \"\"\"\n    Aggregates tensors from a single-level generator into a single tensor by\n    concatenating them. This method is a straightforward way to combine a sequence\n    of tensors into one larger tensor.\n\n    Args:\n        tensor_generator: A generator that yields `torch.Tensor` objects.\n\n    Returns:\n        A single tensor formed by concatenating all tensors from the generator.\n            The concatenation is performed along the default dimension (0).\n    \"\"\"\n    return torch.cat(list(tensor_generator))\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.NestedTorchCatAggregator","title":"NestedTorchCatAggregator","text":"<p>             Bases: <code>NestedSequenceAggregator[Tensor]</code></p> <p>An aggregator that concatenates tensors using PyTorch's torch.cat function. Concatenation is done along the first two dimensions of the chunks.</p>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.NestedTorchCatAggregator.__call__","title":"__call__","text":"<pre><code>__call__(\n    nested_generators_of_tensors: Generator[\n        Generator[Tensor, None, None], None, None\n    ]\n)\n</code></pre> <p>Aggregates tensors from a nested generator structure into a single tensor by concatenating. Each inner generator is first concatenated along dimension 1 into a tensor, and then these tensors are concatenated along dimension 0 together to form the final tensor.</p> PARAMETER  DESCRIPTION <code>nested_generators_of_tensors</code> <p>A generator of generators, where each inner generator yields <code>torch.Tensor</code> objects.</p> <p> TYPE: <code>Generator[Generator[Tensor, None, None], None, None]</code> </p> RETURNS DESCRIPTION <p>A single tensor formed by concatenating all tensors from the nested</p> <p>generators.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def __call__(\n    self,\n    nested_generators_of_tensors: Generator[\n        Generator[torch.Tensor, None, None], None, None\n    ],\n):\n    \"\"\"\n    Aggregates tensors from a nested generator structure into a single tensor by\n    concatenating. Each inner generator is first concatenated along dimension 1 into\n    a tensor, and then these tensors are concatenated along dimension 0 together to\n    form the final tensor.\n\n    Args:\n        nested_generators_of_tensors: A generator of generators, where each inner\n            generator yields `torch.Tensor` objects.\n\n    Returns:\n        A single tensor formed by concatenating all tensors from the nested\n        generators.\n\n    \"\"\"\n    return torch.cat(\n        list(\n            map(\n                lambda tensor_gen: torch.cat(list(tensor_gen), dim=1),\n                nested_generators_of_tensors,\n            )\n        )\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.EkfacRepresentation","title":"EkfacRepresentation  <code>dataclass</code>","text":"<pre><code>EkfacRepresentation(\n    layer_names: Iterable[str],\n    layers_module: Iterable[Module],\n    evecs_a: Iterable[Tensor],\n    evecs_g: Iterable[Tensor],\n    diags: Iterable[Tensor],\n)\n</code></pre> <p>Container class for the EKFAC representation of the Hessian. It can be iterated over to get the layers names and their corresponding module, eigenvectors and diagonal elements of the factorized Hessian matrix.</p> PARAMETER  DESCRIPTION <code>layer_names</code> <p>Names of the layers.</p> <p> TYPE: <code>Iterable[str]</code> </p> <code>layers_module</code> <p>The layers.</p> <p> TYPE: <code>Iterable[Module]</code> </p> <code>evecs_a</code> <p>The a eigenvectors of the ekfac representation.</p> <p> TYPE: <code>Iterable[Tensor]</code> </p> <code>evecs_g</code> <p>The g eigenvectors of the ekfac representation.</p> <p> TYPE: <code>Iterable[Tensor]</code> </p> <code>diags</code> <p>The diagonal elements of the factorized Hessian matrix.</p> <p> TYPE: <code>Iterable[Tensor]</code> </p>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.EkfacRepresentation.get_layer_evecs","title":"get_layer_evecs","text":"<pre><code>get_layer_evecs() -&gt; Tuple[Dict[str, Tensor], Dict[str, Tensor]]\n</code></pre> <p>It returns two dictionaries, one for the a eigenvectors and one for the g eigenvectors, with the layer names as keys. The eigenvectors are in the same order as the layers in the model.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def get_layer_evecs(\n    self,\n) -&gt; Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]]:\n    \"\"\"\n    It returns two dictionaries, one for the a eigenvectors and one for the g\n    eigenvectors, with the layer names as keys. The eigenvectors are in the same\n    order as the layers in the model.\n    \"\"\"\n    evecs_a_dict = {layer_name: evec_a for layer_name, (_, evec_a, _, _) in self}\n    evecs_g_dict = {layer_name: evec_g for layer_name, (_, _, evec_g, _) in self}\n    return evecs_a_dict, evecs_g_dict\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.to_model_device","title":"to_model_device","text":"<pre><code>to_model_device(x: Tensor, model: Module) -&gt; Tensor\n</code></pre> <p>Returns the tensor <code>x</code> moved to the device of the <code>model</code>, if device of model is set</p> PARAMETER  DESCRIPTION <code>x</code> <p>The tensor to be moved to the device of the model.</p> <p> TYPE: <code>Tensor</code> </p> <code>model</code> <p>The model whose device will be used to move the tensor.</p> <p> TYPE: <code>Module</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>The tensor <code>x</code> moved to the device of the <code>model</code>, if device of model is set.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def to_model_device(x: torch.Tensor, model: torch.nn.Module) -&gt; torch.Tensor:\n    \"\"\"\n    Returns the tensor `x` moved to the device of the `model`, if device of model is set\n\n    Args:\n        x: The tensor to be moved to the device of the model.\n        model: The model whose device will be used to move the tensor.\n\n    Returns:\n        The tensor `x` moved to the device of the `model`, if device of model is set.\n    \"\"\"\n    device = next(model.parameters()).device\n    return x.to(device)\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.reshape_vector_to_tensors","title":"reshape_vector_to_tensors","text":"<pre><code>reshape_vector_to_tensors(\n    input_vector: Tensor, target_shapes: Iterable[Tuple[int, ...]]\n) -&gt; Tuple[Tensor, ...]\n</code></pre> <p>Reshape a 1D tensor into multiple tensors with specified shapes.</p> <p>This function takes a 1D tensor (input_vector) and reshapes it into a series of tensors with shapes given by 'target_shapes'. The reshaped tensors are returned as a tuple in the same order as their corresponding shapes.</p> Note <p>The total number of elements in 'input_vector' must be equal to the     sum of the products of the shapes in 'target_shapes'.</p> PARAMETER  DESCRIPTION <code>input_vector</code> <p>The 1D tensor to be reshaped. Must be 1D.</p> <p> TYPE: <code>Tensor</code> </p> <code>target_shapes</code> <p>An iterable of tuples. Each tuple defines the shape of a tensor to be reshaped from the 'input_vector'.</p> <p> TYPE: <code>Iterable[Tuple[int, ...]]</code> </p> RETURNS DESCRIPTION <code>Tuple[Tensor, ...]</code> <p>A tuple of reshaped tensors.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If 'input_vector' is not a 1D tensor or if the total number of elements in 'input_vector' does not match the sum of the products of the shapes in 'target_shapes'.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def reshape_vector_to_tensors(\n    input_vector: torch.Tensor, target_shapes: Iterable[Tuple[int, ...]]\n) -&gt; Tuple[torch.Tensor, ...]:\n    \"\"\"\n    Reshape a 1D tensor into multiple tensors with specified shapes.\n\n    This function takes a 1D tensor (input_vector) and reshapes it into a series of\n    tensors with shapes given by 'target_shapes'.\n    The reshaped tensors are returned as a tuple in the same order\n    as their corresponding shapes.\n\n    Note:\n        The total number of elements in 'input_vector' must be equal to the\n            sum of the products of the shapes in 'target_shapes'.\n\n    Args:\n        input_vector: The 1D tensor to be reshaped. Must be 1D.\n        target_shapes: An iterable of tuples. Each tuple defines the shape of a tensor\n            to be reshaped from the 'input_vector'.\n\n    Returns:\n        A tuple of reshaped tensors.\n\n    Raises:\n        ValueError: If 'input_vector' is not a 1D tensor or if the total\n            number of elements in 'input_vector' does not\n            match the sum of the products of the shapes in 'target_shapes'.\n    \"\"\"\n\n    if input_vector.dim() != 1:\n        raise ValueError(\"Input vector must be a 1D tensor\")\n\n    total_elements = sum(math.prod(shape) for shape in target_shapes)\n\n    if total_elements != input_vector.shape[0]:\n        raise ValueError(\n            f\"The total elements in shapes {total_elements} \"\n            f\"does not match the vector length {input_vector.shape[0]}\"\n        )\n\n    tensors = []\n    start = 0\n    for shape in target_shapes:\n        size = math.prod(shape)  # compute the total size of the tensor with this shape\n        tensors.append(\n            input_vector[start : start + size].view(shape)\n        )  # slice the vector and reshape it\n        start += size\n    return tuple(tensors)\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.align_structure","title":"align_structure","text":"<pre><code>align_structure(\n    source: Mapping[str, Tensor], target: TorchTensorContainerType\n) -&gt; Dict[str, Tensor]\n</code></pre> <p>This function transforms <code>target</code> to have the same structure as <code>source</code>, i.e., it should be a dictionary with the same keys as <code>source</code> and each corresponding value in <code>target</code> should have the same shape as the value in <code>source</code>.</p> PARAMETER  DESCRIPTION <code>source</code> <p>The reference dictionary containing PyTorch tensors.</p> <p> TYPE: <code>Mapping[str, Tensor]</code> </p> <code>target</code> <p>The input to be harmonized. It can be a dictionary, tuple, or tensor.</p> <p> TYPE: <code>TorchTensorContainerType</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>The harmonized version of <code>target</code>.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>target</code> cannot be harmonized to match <code>source</code>.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def align_structure(\n    source: Mapping[str, torch.Tensor],\n    target: TorchTensorContainerType,\n) -&gt; Dict[str, torch.Tensor]:\n    \"\"\"\n    This function transforms `target` to have the same structure as `source`, i.e.,\n    it should be a dictionary with the same keys as `source` and each corresponding\n    value in `target` should have the same shape as the value in `source`.\n\n    Args:\n        source: The reference dictionary containing PyTorch tensors.\n        target: The input to be harmonized. It can be a dictionary, tuple, or tensor.\n\n    Returns:\n        The harmonized version of `target`.\n\n    Raises:\n        ValueError: If `target` cannot be harmonized to match `source`.\n    \"\"\"\n\n    tangent_dict: Dict[str, torch.Tensor]\n\n    if isinstance(target, dict):\n        if list(target.keys()) != list(source.keys()):\n            raise ValueError(\"The keys in 'target' do not match the keys in 'source'.\")\n\n        if [v.shape for v in target.values()] != [v.shape for v in source.values()]:\n            raise ValueError(\n                \"The shapes of the values in 'target' do not match the shapes \"\n                \"of the values in 'source'.\"\n            )\n\n        tangent_dict = target\n\n    elif isinstance(target, tuple) or isinstance(target, list):\n        if [v.shape for v in target] != [v.shape for v in source.values()]:\n            raise ValueError(\n                \"'target' is a tuple/list but its elements' shapes do not match \"\n                \"the shapes of the values in 'source'.\"\n            )\n\n        tangent_dict = dict(zip(source.keys(), target))\n\n    elif isinstance(target, torch.Tensor):\n        try:\n            tangent_dict = dict(\n                zip(\n                    source.keys(),\n                    reshape_vector_to_tensors(\n                        target, [p.shape for p in source.values()]\n                    ),\n                )\n            )\n        except Exception as e:\n            raise ValueError(\n                f\"'target' is a tensor but cannot be reshaped to match 'source'. \"\n                f\"Original error: {e}\"\n            )\n\n    else:\n        raise ValueError(f\"'target' is of type {type(target)} which is not supported.\")\n\n    return tangent_dict\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.align_with_model","title":"align_with_model","text":"<pre><code>align_with_model(x: TorchTensorContainerType, model: Module)\n</code></pre> <p>Aligns an input to the model's parameter structure, i.e. transforms it into a dict with the same keys as model.named_parameters() and matching tensor shapes</p> PARAMETER  DESCRIPTION <code>x</code> <p>The input to be aligned. It can be a dictionary, tuple, or tensor.</p> <p> TYPE: <code>TorchTensorContainerType</code> </p> <code>model</code> <p>model to use for alignment</p> <p> TYPE: <code>Module</code> </p> RETURNS DESCRIPTION <p>The aligned version of <code>x</code>.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>x</code> cannot be aligned to match the model's parameters .</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def align_with_model(x: TorchTensorContainerType, model: torch.nn.Module):\n    \"\"\"\n    Aligns an input to the model's parameter structure, i.e. transforms it into a dict\n    with the same keys as model.named_parameters() and matching tensor shapes\n\n    Args:\n        x: The input to be aligned. It can be a dictionary, tuple, or tensor.\n        model: model to use for alignment\n\n    Returns:\n        The aligned version of `x`.\n\n    Raises:\n        ValueError: If `x` cannot be aligned to match the model's parameters .\n\n    \"\"\"\n    model_params = {k: p for k, p in model.named_parameters() if p.requires_grad}\n    return align_structure(model_params, x)\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.flatten_dimensions","title":"flatten_dimensions","text":"<pre><code>flatten_dimensions(\n    tensors: Iterable[Tensor],\n    shape: Optional[Tuple[int, ...]] = None,\n    concat_at: int = -1,\n) -&gt; Tensor\n</code></pre> <p>Flattens the dimensions of each tensor in the given iterable and concatenates them along a specified dimension.</p> <p>This function takes an iterable of PyTorch tensors and flattens each tensor. Optionally, each tensor can be reshaped to a specified shape before concatenation. The concatenation is performed along the dimension specified by <code>concat_at</code>.</p> PARAMETER  DESCRIPTION <code>tensors</code> <p>An iterable containing PyTorch tensors to be flattened and concatenated.</p> <p> TYPE: <code>Iterable[Tensor]</code> </p> <code>shape</code> <p>A tuple representing the desired shape to which each tensor is reshaped before concatenation. If None, tensors are flattened to 1D.</p> <p> TYPE: <code>Optional[Tuple[int, ...]]</code> DEFAULT: <code>None</code> </p> <code>concat_at</code> <p>The dimension along which to concatenate the tensors.</p> <p> TYPE: <code>int</code> DEFAULT: <code>-1</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A single tensor resulting from the concatenation of the input tensors,</p> <code>Tensor</code> <p>each either flattened or reshaped as specified.</p> Example <pre><code>&gt;&gt;&gt; tensors = [torch.tensor([[1, 2], [3, 4]]), torch.tensor([[5, 6], [7, 8]])]\n&gt;&gt;&gt; flatten_dimensions(tensors)\ntensor([1, 2, 3, 4, 5, 6, 7, 8])\n\n&gt;&gt;&gt; flatten_dimensions(tensors, shape=(2, 2), concat_at=0)\ntensor([[1, 2],\n        [3, 4],\n        [5, 6],\n        [7, 8]])\n</code></pre> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def flatten_dimensions(\n    tensors: Iterable[torch.Tensor],\n    shape: Optional[Tuple[int, ...]] = None,\n    concat_at: int = -1,\n) -&gt; torch.Tensor:\n    \"\"\"\n    Flattens the dimensions of each tensor in the given iterable and concatenates them\n    along a specified dimension.\n\n    This function takes an iterable of PyTorch tensors and flattens each tensor.\n    Optionally, each tensor can be reshaped to a specified shape before concatenation.\n    The concatenation is performed along the dimension specified by `concat_at`.\n\n    Args:\n        tensors: An iterable containing PyTorch tensors to be flattened\n            and concatenated.\n        shape: A tuple representing the desired shape to which each tensor is reshaped\n            before concatenation. If None, tensors are flattened to 1D.\n        concat_at: The dimension along which to concatenate the tensors.\n\n    Returns:\n        A single tensor resulting from the concatenation of the input tensors,\n        each either flattened or reshaped as specified.\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; tensors = [torch.tensor([[1, 2], [3, 4]]), torch.tensor([[5, 6], [7, 8]])]\n        &gt;&gt;&gt; flatten_dimensions(tensors)\n        tensor([1, 2, 3, 4, 5, 6, 7, 8])\n\n        &gt;&gt;&gt; flatten_dimensions(tensors, shape=(2, 2), concat_at=0)\n        tensor([[1, 2],\n                [3, 4],\n                [5, 6],\n                [7, 8]])\n        ```\n    \"\"\"\n    return torch.cat(\n        [t.reshape(-1) if shape is None else t.reshape(*shape) for t in tensors],\n        dim=concat_at,\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.torch_dataset_to_dask_array","title":"torch_dataset_to_dask_array","text":"<pre><code>torch_dataset_to_dask_array(\n    dataset: Dataset,\n    chunk_size: int,\n    total_size: Optional[int] = None,\n    resulting_dtype: Type[number] = np.float32,\n) -&gt; Tuple[Array, ...]\n</code></pre> <p>Construct tuple of dask arrays from a PyTorch dataset, using dask.delayed</p> PARAMETER  DESCRIPTION <code>dataset</code> <p>A PyTorch dataset</p> <p> TYPE: <code>Dataset</code> </p> <code>chunk_size</code> <p>The size of the chunks for the resulting Dask arrays.</p> <p> TYPE: <code>int</code> </p> <code>total_size</code> <p>If the dataset does not implement len, provide the length via this parameter. If None the length of the dataset is inferred via accessing the dataset once.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>resulting_dtype</code> <p>The dtype of the resulting dask.array.Array</p> <p> TYPE: <code>Type[number]</code> DEFAULT: <code>float32</code> </p> Example <pre><code>import torch\nfrom torch.utils.data import TensorDataset\nx = torch.rand((20, 3))\ny = torch.rand((20, 1))\ndataset = TensorDataset(x, y)\nda_x, da_y = torch_dataset_to_dask_array(dataset, 4)\n</code></pre> RETURNS DESCRIPTION <code>Tuple[Array, ...]</code> <p>Tuple of Dask arrays corresponding to each tensor in the dataset.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def torch_dataset_to_dask_array(\n    dataset: Dataset,\n    chunk_size: int,\n    total_size: Optional[int] = None,\n    resulting_dtype: Type[np.number] = np.float32,\n) -&gt; Tuple[da.Array, ...]:\n    \"\"\"\n    Construct tuple of dask arrays from a PyTorch dataset, using dask.delayed\n\n    Args:\n        dataset: A PyTorch [dataset][torch.utils.data.Dataset]\n        chunk_size: The size of the chunks for the resulting Dask arrays.\n        total_size: If the dataset does not implement len, provide the length\n            via this parameter. If None\n            the length of the dataset is inferred via accessing the dataset once.\n        resulting_dtype: The dtype of the resulting [dask.array.Array][dask.array.Array]\n\n    ??? Example\n        ```python\n        import torch\n        from torch.utils.data import TensorDataset\n        x = torch.rand((20, 3))\n        y = torch.rand((20, 1))\n        dataset = TensorDataset(x, y)\n        da_x, da_y = torch_dataset_to_dask_array(dataset, 4)\n        ```\n\n    Returns:\n        Tuple of Dask arrays corresponding to each tensor in the dataset.\n    \"\"\"\n\n    def _infer_data_len(d_set: Dataset):\n        try:\n            n_data = len(d_set)\n            if total_size is not None and n_data != total_size:\n                raise ValueError(\n                    f\"The number of samples in the dataset ({n_data}), derived \"\n                    f\"from calling \u00b4len\u00b4, does not match the provided \"\n                    f\"total number of samples ({total_size}). \"\n                    f\"Call the function without total_size.\"\n                )\n            return n_data\n        except TypeError as e:\n            err_msg = (\n                f\"Could not infer the number of samples in the dataset from \"\n                f\"calling \u00b4len\u00b4. Original error: {e}.\"\n            )\n            if total_size is not None:\n                logger.warning(\n                    err_msg\n                    + f\" Using the provided total number of samples {total_size}.\"\n                )\n                return total_size\n            else:\n                logger.warning(\n                    err_msg + f\" Infer the number of samples from the dataset, \"\n                    f\"via iterating the dataset once. \"\n                    f\"This might induce severe overhead, so consider\"\n                    f\"providing total_size, if you know the number of samples \"\n                    f\"beforehand.\"\n                )\n                idx = 0\n                while True:\n                    try:\n                        t = d_set[idx]\n                        if all(_t.numel() == 0 for _t in t):\n                            return idx\n                        idx += 1\n\n                    except IndexError:\n                        return idx\n\n    sample = dataset[0]\n    if not isinstance(sample, tuple):\n        sample = (sample,)\n\n    def _get_chunk(\n        start_idx: int, stop_idx: int, d_set: Dataset\n    ) -&gt; Tuple[torch.Tensor, ...]:\n        try:\n            t = d_set[start_idx:stop_idx]\n            if not isinstance(t, tuple):\n                t = (t,)\n            return t  # type:ignore\n        except Exception:\n            nested_tensor_list = [\n                [d_set[idx][k] for idx in range(start_idx, stop_idx)]\n                for k in range(len(sample))\n            ]\n            return tuple(map(torch.stack, nested_tensor_list))\n\n    n_samples = _infer_data_len(dataset)\n    chunk_indices = [\n        (i, min(i + chunk_size, n_samples)) for i in range(0, n_samples, chunk_size)\n    ]\n    delayed_dataset = dask.delayed(dataset)\n    delayed_chunks = [\n        dask.delayed(partial(_get_chunk, start, stop))(delayed_dataset)\n        for (start, stop) in chunk_indices\n    ]\n\n    delayed_arrays_dict: Dict[int, List[da.Array]] = {k: [] for k in range(len(sample))}\n\n    for chunk, (start, stop) in zip(delayed_chunks, chunk_indices):\n        for tensor_idx, sample_tensor in enumerate(sample):\n            delayed_tensor = da.from_delayed(\n                dask.delayed(lambda t: t.cpu().numpy())(chunk[tensor_idx]),\n                shape=(stop - start, *sample_tensor.shape),\n                dtype=resulting_dtype,\n            )\n\n            delayed_arrays_dict[tensor_idx].append(delayed_tensor)\n\n    return tuple(\n        da.concatenate(array_list) for array_list in delayed_arrays_dict.values()\n    )\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.empirical_cross_entropy_loss_fn","title":"empirical_cross_entropy_loss_fn","text":"<pre><code>empirical_cross_entropy_loss_fn(\n    model_output: Tensor, *args, **kwargs\n) -&gt; Tensor\n</code></pre> <p>Computes the empirical cross entropy loss of the model output. This is the cross entropy loss of the model output without the labels. The function takes all the usual arguments and keyword arguments of the cross entropy loss function, so that it is compatible with the PyTorch cross entropy loss function. However, it ignores everything except the first argument, which is the model output.</p> PARAMETER  DESCRIPTION <code>model_output</code> <p>The output of the model.</p> <p> TYPE: <code>Tensor</code> </p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def empirical_cross_entropy_loss_fn(\n    model_output: torch.Tensor, *args, **kwargs\n) -&gt; torch.Tensor:\n    \"\"\"\n    Computes the empirical cross entropy loss of the model output. This is the\n    cross entropy loss of the model output without the labels. The function takes\n    all the usual arguments and keyword arguments of the cross entropy loss\n    function, so that it is compatible with the PyTorch cross entropy loss\n    function. However, it ignores everything except the first argument, which is\n    the model output.\n\n    Args:\n        model_output: The output of the model.\n    \"\"\"\n    probs_ = torch.softmax(model_output, dim=1)\n    log_probs_ = torch.log(probs_)\n    log_probs_ = torch.where(\n        torch.isfinite(log_probs_), log_probs_, torch.zeros_like(log_probs_)\n    )\n    return torch.sum(log_probs_ * probs_.detach() ** 0.5)\n</code></pre>"},{"location":"api/pydvl/parallel/","title":"Parallel","text":""},{"location":"api/pydvl/parallel/#pydvl.parallel","title":"pydvl.parallel","text":"<p>This module provides a common interface to parallelization backends. The list of supported backends is here. Backends should be instantiated directly and passed to the respective valuation method.</p> <p>We use executors that implement the Executor interface to submit tasks in parallel. The basic high-level pattern is:</p> <pre><code>from pydvl.parallel import JoblibParallelBackend\n\nparallel_backend = JoblibParallelBackend()\nwith parallel_backend.executor(max_workers=2) as executor:\n    future = executor.submit(lambda x: x + 1, 1)\n    result = future.result()\nassert result == 2\n</code></pre> <p>Running a map-style job is also easy:</p> <pre><code>from pydvl.parallel import JoblibParallelBackend\n\nparallel_backend = JoblibParallelBackend()\nwith parallel_backend.executor(max_workers=2) as executor:\n    results = list(executor.map(lambda x: x + 1, range(5)))\nassert results == [1, 2, 3, 4, 5]\n</code></pre> <p>Passsing large objects</p> <p>When running tasks which accept heavy inputs, it is important to first use <code>put()</code> on the object and use the returned reference as argument to the callable within <code>submit()</code>. For example:   <pre><code>u_ref = parallel_backend.put(u)\n...\nexecutor.submit(task, utility=u)\n</code></pre> Note that <code>task()</code> does not need to be changed in any way: the backend will <code>get()</code> the object and pass it to the function upon invocation.</p> <p>There is an alternative map-reduce implementation MapReduceJob which internally uses joblib's higher level API with <code>Parallel()</code> which then indirectly also supports the use of Dask and Ray.</p>"},{"location":"api/pydvl/parallel/backend/","title":"Backend","text":""},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend","title":"pydvl.parallel.backend","text":""},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.CancellationPolicy","title":"CancellationPolicy","text":"<p>             Bases: <code>Flag</code></p> <p>Policy to use when cancelling futures after exiting an Executor.</p> <p>Note</p> <p>Not all backends support all policies.</p> ATTRIBUTE DESCRIPTION <code>NONE</code> <p>Do not cancel any futures.</p> <p> </p> <code>PENDING</code> <p>Cancel all pending futures, but not running ones.</p> <p> </p> <code>RUNNING</code> <p>Cancel all running futures, but not pending ones.</p> <p> </p> <code>ALL</code> <p>Cancel all pending and running futures.</p> <p> </p>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.ParallelBackend","title":"ParallelBackend","text":"<p>Abstract base class for all parallel backends.</p>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.ParallelBackend.executor","title":"executor  <code>abstractmethod</code> <code>classmethod</code>","text":"<pre><code>executor(\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.PENDING\n) -&gt; Executor\n</code></pre> <p>Returns a futures executor for the parallel backend.</p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>@classmethod\n@abstractmethod\ndef executor(\n    cls,\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.PENDING,\n) -&gt; Executor:\n    \"\"\"Returns a futures executor for the parallel backend.\"\"\"\n    ...\n</code></pre>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.init_parallel_backend","title":"init_parallel_backend","text":"<pre><code>init_parallel_backend(\n    config: ParallelConfig | None = None, backend_name: str | None = None\n) -&gt; ParallelBackend\n</code></pre> <p>Initializes the parallel backend and returns an instance of it.</p> <p>The following example creates a parallel backend instance with the default configuration, which is a local joblib backend.</p> <p>If you don't pass any arguments, then by default it will instantiate the JoblibParallelBackend:</p> Example <pre><code>parallel_backend = init_parallel_backend()\n</code></pre> <p>To create a parallel backend instance with for example <code>ray</code> as a backend, you can pass the backend name as a string:.</p> Example <pre><code>parallel_backend = init_parallel_backend(backend_name=\"ray\")\n</code></pre> <p>The following is an example of the deprecated way for instantiating a parallel backend:</p> Example <pre><code>config = ParallelConfig()\nparallel_backend = init_parallel_backend(config)\n</code></pre> PARAMETER  DESCRIPTION <code>backend_name</code> <p>Name of the backend to instantiate.</p> <p> TYPE: <code>str | None</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig | None</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef init_parallel_backend(\n    config: ParallelConfig | None = None, backend_name: str | None = None\n) -&gt; ParallelBackend:\n    \"\"\"Initializes the parallel backend and returns an instance of it.\n\n    The following example creates a parallel backend instance with the default\n    configuration, which is a local joblib backend.\n\n    If you don't pass any arguments, then by default it will instantiate\n    the JoblibParallelBackend:\n\n    ??? Example\n        ```python\n        parallel_backend = init_parallel_backend()\n        ```\n\n    To create a parallel backend instance with for example `ray` as a backend,\n    you can pass the backend name as a string:.\n\n    ??? Example\n        ```python\n        parallel_backend = init_parallel_backend(backend_name=\"ray\")\n        ```\n\n\n    The following is an example of the deprecated\n    way for instantiating a parallel backend:\n\n    ??? Example\n        ``` python\n        config = ParallelConfig()\n        parallel_backend = init_parallel_backend(config)\n        ```\n\n    Args:\n        backend_name: Name of the backend to instantiate.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n\n\n    \"\"\"\n    if backend_name is None:\n        if config is None:\n            backend_name = \"joblib\"\n        else:\n            backend_name = config.backend\n\n    try:\n        parallel_backend_cls = ParallelBackend.BACKENDS[backend_name]\n    except KeyError:\n        raise NotImplementedError(f\"Unexpected parallel backend {backend_name}\")\n    return parallel_backend_cls(config)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.available_cpus","title":"available_cpus","text":"<pre><code>available_cpus() -&gt; int\n</code></pre> <p>Platform-independent count of available cores.</p> <p>FIXME: do we really need this or is <code>os.cpu_count</code> enough? Is this portable?</p> RETURNS DESCRIPTION <code>int</code> <p>Number of cores, or 1 if it is not possible to determine.</p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>def available_cpus() -&gt; int:\n    \"\"\"Platform-independent count of available cores.\n\n    FIXME: do we really need this or is `os.cpu_count` enough? Is this portable?\n\n    Returns:\n        Number of cores, or 1 if it is not possible to determine.\n    \"\"\"\n    from platform import system\n\n    if system() != \"Linux\":\n        return os.cpu_count() or 1\n    return len(os.sched_getaffinity(0))  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/config/","title":"Config","text":""},{"location":"api/pydvl/parallel/config/#pydvl.parallel.config","title":"pydvl.parallel.config","text":""},{"location":"api/pydvl/parallel/config/#pydvl.parallel.config.ParallelConfig","title":"ParallelConfig  <code>dataclass</code>","text":"<pre><code>ParallelConfig(\n    backend: Literal[\"joblib\", \"ray\"] = \"joblib\",\n    address: Optional[Union[str, Tuple[str, int]]] = None,\n    n_cpus_local: Optional[int] = None,\n    logging_level: Optional[int] = None,\n    wait_timeout: float = 1.0,\n)\n</code></pre> <p>Configuration for parallel computation backend.</p> PARAMETER  DESCRIPTION <code>backend</code> <p>Type of backend to use. Defaults to 'joblib'</p> <p> TYPE: <code>Literal['joblib', 'ray']</code> DEFAULT: <code>'joblib'</code> </p> <code>address</code> <p>(DEPRECATED) Address of existing remote or local cluster to use.</p> <p> TYPE: <code>Optional[Union[str, Tuple[str, int]]]</code> DEFAULT: <code>None</code> </p> <code>n_cpus_local</code> <p>(DEPRECATED) Number of CPUs to use when creating a local ray cluster. This has no effect when using an existing ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>logging_level</code> <p>(DEPRECATED) Logging level for the parallel backend's worker.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>wait_timeout</code> <p>(DEPRECATED) Timeout in seconds for waiting on futures.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p>"},{"location":"api/pydvl/parallel/map_reduce/","title":"Map reduce","text":""},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce","title":"pydvl.parallel.map_reduce","text":"<p>This module contains a wrapper around joblib's <code>Parallel()</code> class that makes it easy to run map-reduce jobs.</p> <p>Deprecation</p> <p>This interface might be deprecated or changed in a future release before 1.0</p>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob","title":"MapReduceJob","text":"<pre><code>MapReduceJob(\n    inputs: Union[Collection[T], T],\n    map_func: MapFunction[R],\n    reduce_func: ReduceFunction[R] = identity,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    *,\n    map_kwargs: Optional[Dict] = None,\n    reduce_kwargs: Optional[Dict] = None,\n    n_jobs: int = -1,\n    timeout: Optional[float] = None\n)\n</code></pre> <p>             Bases: <code>Generic[T, R]</code></p> <p>Takes an embarrassingly parallel fun and runs it in <code>n_jobs</code> parallel jobs, splitting the data evenly into a number of chunks equal to the number of jobs.</p> <p>Typing information for objects of this class requires the type of the inputs that are split for <code>map_func</code> and the type of its output.</p> PARAMETER  DESCRIPTION <code>inputs</code> <p>The input that will be split and passed to <code>map_func</code>. if it's not a sequence object. It will be repeat <code>n_jobs</code> number of times.</p> <p> TYPE: <code>Union[Collection[T], T]</code> </p> <code>map_func</code> <p>Function that will be applied to the input chunks in each job.</p> <p> TYPE: <code>MapFunction[R]</code> </p> <code>reduce_func</code> <p>Function that will be applied to the results of <code>map_func</code> to reduce them.</p> <p> TYPE: <code>ReduceFunction[R]</code> DEFAULT: <code>identity</code> </p> <code>map_kwargs</code> <p>Keyword arguments that will be passed to <code>map_func</code> in each job. Alternatively, one can use functools.partial.</p> <p> TYPE: <code>Optional[Dict]</code> DEFAULT: <code>None</code> </p> <code>reduce_kwargs</code> <p>Keyword arguments that will be passed to <code>reduce_func</code> in each job. Alternatively, one can use functools.partial.</p> <p> TYPE: <code>Optional[Dict]</code> DEFAULT: <code>None</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to run. Does not accept 0</p> <p> TYPE: <code>int</code> DEFAULT: <code>-1</code> </p> Example <p>A simple usage example with 2 jobs:</p> <pre><code>&gt;&gt;&gt; from pydvl.parallel import MapReduceJob\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; map_reduce_job: MapReduceJob[np.ndarray, np.ndarray] = MapReduceJob(\n...     np.arange(5),\n...     map_func=np.sum,\n...     reduce_func=np.sum,\n...     n_jobs=2,\n... )\n&gt;&gt;&gt; map_reduce_job()\n10\n</code></pre> <p>When passed a single object as input, it will be repeated for each job: <pre><code>&gt;&gt;&gt; from pydvl.parallel import MapReduceJob\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; map_reduce_job: MapReduceJob[int, np.ndarray] = MapReduceJob(\n...     5,\n...     map_func=lambda x: np.array([x]),\n...     reduce_func=np.sum,\n...     n_jobs=2,\n... )\n&gt;&gt;&gt; map_reduce_job()\n10\n</code></pre></p> Source code in <code>src/pydvl/parallel/map_reduce.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef __init__(\n    self,\n    inputs: Union[Collection[T], T],\n    map_func: MapFunction[R],\n    reduce_func: ReduceFunction[R] = identity,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    *,\n    map_kwargs: Optional[Dict] = None,\n    reduce_kwargs: Optional[Dict] = None,\n    n_jobs: int = -1,\n    timeout: Optional[float] = None,\n):\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    self.parallel_backend = parallel_backend\n\n    self.timeout = timeout\n\n    self._n_jobs = -1\n    # This uses the setter defined below\n    self.n_jobs = n_jobs\n\n    self.inputs_ = inputs\n\n    self.map_kwargs = map_kwargs if map_kwargs is not None else dict()\n    self.reduce_kwargs = reduce_kwargs if reduce_kwargs is not None else dict()\n\n    self._map_func = reduce(maybe_add_argument, [\"job_id\", \"seed\"], map_func)\n    self._reduce_func = reduce_func\n</code></pre>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob.n_jobs","title":"n_jobs  <code>property</code> <code>writable</code>","text":"<pre><code>n_jobs: int\n</code></pre> <p>Effective number of jobs according to the used ParallelBackend instance.</p>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob.__call__","title":"__call__","text":"<pre><code>__call__(seed: Optional[Union[Seed, SeedSequence]] = None) -&gt; R\n</code></pre> <p>Runs the map-reduce job.</p> PARAMETER  DESCRIPTION <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Union[Seed, SeedSequence]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>R</code> <p>The result of the reduce function.</p> Source code in <code>src/pydvl/parallel/map_reduce.py</code> <pre><code>def __call__(\n    self,\n    seed: Optional[Union[Seed, SeedSequence]] = None,\n) -&gt; R:\n    \"\"\"\n    Runs the map-reduce job.\n\n    Args:\n        seed: Either an instance of a numpy random number generator or a seed for\n            it.\n\n    Returns:\n         The result of the reduce function.\n    \"\"\"\n    seed_seq = ensure_seed_sequence(seed)\n\n    if hasattr(self.parallel_backend, \"_joblib_backend_name\"):\n        backend = getattr(self.parallel_backend, \"_joblib_backend_name\")\n    else:\n        warnings.warn(\n            \"Parallel backend \"\n            f\"{self.parallel_backend.__class__.__name__}. \"\n            \"should have a `_joblib_backend_name` attribute in order to work \"\n            \"property with MapReduceJob. \"\n            \"Defaulting to joblib loky backend\"\n        )\n        backend = \"loky\"\n\n    with Parallel(backend=backend, prefer=\"processes\") as parallel:\n        chunks = self._chunkify(self.inputs_, n_chunks=self.n_jobs)\n        map_results: List[R] = parallel(\n            delayed(self._map_func)(\n                next_chunk, job_id=j, seed=seed, **self.map_kwargs\n            )\n            for j, (next_chunk, seed) in enumerate(\n                zip(chunks, seed_seq.spawn(len(chunks)))\n            )\n        )\n\n    reduce_results: R = self._reduce_func(map_results, **self.reduce_kwargs)\n    return reduce_results\n</code></pre>"},{"location":"api/pydvl/parallel/backends/","title":"Backends","text":""},{"location":"api/pydvl/parallel/backends/#pydvl.parallel.backends","title":"pydvl.parallel.backends","text":""},{"location":"api/pydvl/parallel/backends/joblib/","title":"Joblib","text":""},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib","title":"pydvl.parallel.backends.joblib","text":""},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib.JoblibParallelBackend","title":"JoblibParallelBackend","text":"<pre><code>JoblibParallelBackend(config: ParallelConfig | None = None)\n</code></pre> <p>             Bases: <code>ParallelBackend</code></p> <p>Class used to wrap joblib to make it transparent to algorithms.</p> <p>Example</p> <pre><code>from pydvl.parallel import JoblibParallelBackend\nparallel_backend = JoblibParallelBackend()\n</code></pre> Source code in <code>src/pydvl/parallel/backends/joblib.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": None},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef __init__(self, config: ParallelConfig | None = None) -&gt; None:\n    n_jobs: int | None = None\n    if config is not None:\n        n_jobs = config.n_cpus_local\n    self.config = {\n        \"n_jobs\": n_jobs,\n    }\n</code></pre>"},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib.JoblibParallelBackend.executor","title":"executor  <code>classmethod</code>","text":"<pre><code>executor(\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.NONE\n) -&gt; Executor\n</code></pre> <p>Returns a futures executor for the parallel backend.</p> <p>Example</p> <pre><code>from pydvl.parallel import JoblibParallelBackend\nparallel_backend = JoblibParallelBackend()\nwith parallel_backend.executor() as executor:\n    executor.submit(...)\n</code></pre> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of parallel workers.</p> <p> TYPE: <code>int | None</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig | None</code> DEFAULT: <code>None</code> </p> <code>cancel_futures</code> <p>Policy to use when cancelling futures after exiting an Executor.</p> <p> TYPE: <code>CancellationPolicy | bool</code> DEFAULT: <code>NONE</code> </p> RETURNS DESCRIPTION <code>Executor</code> <p>Instance of _ReusablePoolExecutor.</p> Source code in <code>src/pydvl/parallel/backends/joblib.py</code> <pre><code>@classmethod\ndef executor(\n    cls,\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.NONE,\n) -&gt; Executor:\n    \"\"\"Returns a futures executor for the parallel backend.\n\n    !!! Example\n        ``` python\n        from pydvl.parallel import JoblibParallelBackend\n        parallel_backend = JoblibParallelBackend()\n        with parallel_backend.executor() as executor:\n            executor.submit(...)\n        ```\n\n    Args:\n        max_workers: Maximum number of parallel workers.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        cancel_futures: Policy to use when cancelling futures\n            after exiting an Executor.\n\n    Returns:\n        Instance of [_ReusablePoolExecutor][joblib.externals.loky.reusable_executor._ReusablePoolExecutor].\n    \"\"\"\n    if config is not None:\n        warnings.warn(\n            \"The `JoblibParallelBackend` uses deprecated arguments: \"\n            \"`config`. They were deprecated since v0.9.0 \"\n            \"and will be removed in v0.10.0.\",\n            FutureWarning,\n        )\n\n    if cancel_futures not in (CancellationPolicy.NONE, False):\n        warnings.warn(\n            \"Cancellation of futures is not supported by the joblib backend\",\n        )\n    return cast(Executor, get_reusable_executor(max_workers=max_workers))\n</code></pre>"},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib.JoblibParallelBackend.wrap","title":"wrap","text":"<pre><code>wrap(fun: Callable, **kwargs) -&gt; Callable\n</code></pre> <p>Wraps a function as a joblib delayed.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>the function to wrap</p> <p> TYPE: <code>Callable</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>The delayed function.</p> Source code in <code>src/pydvl/parallel/backends/joblib.py</code> <pre><code>def wrap(self, fun: Callable, **kwargs) -&gt; Callable:\n    \"\"\"Wraps a function as a joblib delayed.\n\n    Args:\n        fun: the function to wrap\n\n    Returns:\n        The delayed function.\n    \"\"\"\n    return delayed(fun)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backends/ray/","title":"Ray","text":""},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray","title":"pydvl.parallel.backends.ray","text":""},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray.RayParallelBackend","title":"RayParallelBackend","text":"<pre><code>RayParallelBackend(config: ParallelConfig | None = None)\n</code></pre> <p>             Bases: <code>ParallelBackend</code></p> <p>Class used to wrap ray to make it transparent to algorithms.</p> <p>Example</p> <pre><code>import ray\nfrom pydvl.parallel import RayParallelBackend\nray.init()\nparallel_backend = RayParallelBackend()\n</code></pre> Source code in <code>src/pydvl/parallel/backends/ray.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": None},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef __init__(self, config: ParallelConfig | None = None) -&gt; None:\n    if not ray.is_initialized():\n        raise RuntimeError(\n            \"Starting from v0.9.0, ray is no longer automatically initialized. \"\n            \"Please use `ray.init()` with the desired configuration \"\n            \"before using this class.\"\n        )\n    # Register ray joblib backend\n    register_ray()\n</code></pre>"},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray.RayParallelBackend.executor","title":"executor  <code>classmethod</code>","text":"<pre><code>executor(\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.PENDING\n) -&gt; Executor\n</code></pre> <p>Returns a futures executor for the parallel backend.</p> <p>Example</p> <pre><code>import ray\nfrom pydvl.parallel import RayParallelBackend\nray.init()\nparallel_backend = RayParallelBackend()\nwith parallel_backend.executor() as executor:\n    executor.submit(...)\n</code></pre> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of parallel workers.</p> <p> TYPE: <code>int | None</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig | None</code> DEFAULT: <code>None</code> </p> <code>cancel_futures</code> <p>Policy to use when cancelling futures after exiting an Executor.</p> <p> TYPE: <code>CancellationPolicy | bool</code> DEFAULT: <code>PENDING</code> </p> RETURNS DESCRIPTION <code>Executor</code> <p>Instance of RayExecutor.</p> Source code in <code>src/pydvl/parallel/backends/ray.py</code> <pre><code>@classmethod\ndef executor(\n    cls,\n    max_workers: int | None = None,\n    *,\n    config: ParallelConfig | None = None,\n    cancel_futures: CancellationPolicy | bool = CancellationPolicy.PENDING,\n) -&gt; Executor:\n    \"\"\"Returns a futures executor for the parallel backend.\n\n    !!! Example\n        ``` python\n        import ray\n        from pydvl.parallel import RayParallelBackend\n        ray.init()\n        parallel_backend = RayParallelBackend()\n        with parallel_backend.executor() as executor:\n            executor.submit(...)\n        ```\n\n    Args:\n        max_workers: Maximum number of parallel workers.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        cancel_futures: Policy to use when cancelling futures\n            after exiting an Executor.\n\n    Returns:\n        Instance of [RayExecutor][pydvl.parallel.futures.ray.RayExecutor].\n    \"\"\"\n    # Imported here to avoid circular import errors\n    from pydvl.parallel.futures.ray import RayExecutor\n\n    if config is not None:\n        warnings.warn(\n            \"The `RayParallelBackend` uses deprecated arguments: \"\n            \"`config`. They were deprecated since v0.9.0 \"\n            \"and will be removed in v0.10.0.\",\n            FutureWarning,\n        )\n\n    return RayExecutor(max_workers, cancel_futures=cancel_futures)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray.RayParallelBackend.wrap","title":"wrap","text":"<pre><code>wrap(fun: Callable, **kwargs) -&gt; Callable\n</code></pre> <p>Wraps a function as a ray remote.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>the function to wrap</p> <p> TYPE: <code>Callable</code> </p> <code>kwargs</code> <p>keyword arguments to pass to @ray.remote</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>The <code>.remote</code> method of the ray <code>RemoteFunction</code>.</p> Source code in <code>src/pydvl/parallel/backends/ray.py</code> <pre><code>def wrap(self, fun: Callable, **kwargs) -&gt; Callable:\n    \"\"\"Wraps a function as a ray remote.\n\n    Args:\n        fun: the function to wrap\n        kwargs: keyword arguments to pass to @ray.remote\n\n    Returns:\n        The `.remote` method of the ray `RemoteFunction`.\n    \"\"\"\n    if len(kwargs) &gt; 0:\n        return ray.remote(**kwargs)(fun).remote  # type: ignore\n    return ray.remote(fun).remote  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/futures/","title":"Futures","text":""},{"location":"api/pydvl/parallel/futures/#pydvl.parallel.futures","title":"pydvl.parallel.futures","text":""},{"location":"api/pydvl/parallel/futures/#pydvl.parallel.futures.init_executor","title":"init_executor","text":"<pre><code>init_executor(\n    max_workers: Optional[int] = None,\n    config: Optional[ParallelConfig] = None,\n    **kwargs\n) -&gt; Generator[Executor, None, None]\n</code></pre> <p>Initializes a futures executor for the given parallel configuration.</p> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of concurrent tasks.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Other optional parameter that will be passed to the executor.</p> <p> DEFAULT: <code>{}</code> </p> Examples <p><pre><code>from pydvl.parallel.futures import init_executor, ParallelConfig\n\nconfig = ParallelConfig(backend=\"ray\")\nwith init_executor(max_workers=1, config=config) as executor:\n    future = executor.submit(lambda x: x + 1, 1)\n    result = future.result()\nassert result == 2\n</code></pre> <pre><code>from pydvl.parallel.futures import init_executor\nwith init_executor() as executor:\n    results = list(executor.map(lambda x: x + 1, range(5)))\nassert results == [1, 2, 3, 4, 5]\n</code></pre></p> Source code in <code>src/pydvl/parallel/futures/__init__.py</code> <pre><code>@contextmanager\n@deprecated(\n    target=None,\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef init_executor(\n    max_workers: Optional[int] = None,\n    config: Optional[ParallelConfig] = None,\n    **kwargs,\n) -&gt; Generator[Executor, None, None]:\n    \"\"\"Initializes a futures executor for the given parallel configuration.\n\n    Args:\n        max_workers: Maximum number of concurrent tasks.\n        config: instance of [ParallelConfig][pydvl.utils.config.ParallelConfig]\n            with cluster address, number of cpus, etc.\n        kwargs: Other optional parameter that will be passed to the executor.\n\n\n    ??? Examples\n        ``` python\n        from pydvl.parallel.futures import init_executor, ParallelConfig\n\n        config = ParallelConfig(backend=\"ray\")\n        with init_executor(max_workers=1, config=config) as executor:\n            future = executor.submit(lambda x: x + 1, 1)\n            result = future.result()\n        assert result == 2\n        ```\n        ``` python\n        from pydvl.parallel.futures import init_executor\n        with init_executor() as executor:\n            results = list(executor.map(lambda x: x + 1, range(5)))\n        assert results == [1, 2, 3, 4, 5]\n        ```\n    \"\"\"\n\n    if config is None:\n        config = ParallelConfig()\n\n    try:\n        cls = ParallelBackend.BACKENDS[config.backend]\n        with cls.executor(max_workers=max_workers, config=config, **kwargs) as e:\n            yield e\n    except KeyError:\n        raise NotImplementedError(f\"Unexpected parallel backend {config.backend}\")\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/","title":"Ray","text":""},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray","title":"pydvl.parallel.futures.ray","text":""},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor","title":"RayExecutor","text":"<pre><code>RayExecutor(\n    max_workers: Optional[int] = None,\n    *,\n    config: Optional[ParallelConfig] = None,\n    cancel_futures: Union[CancellationPolicy, bool] = CancellationPolicy.ALL\n)\n</code></pre> <p>             Bases: <code>Executor</code></p> <p>Asynchronous executor using Ray that implements the concurrent.futures API.</p> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of concurrent tasks. Each task can request itself any number of vCPUs. You must ensure the product of this value and the n_cpus_per_job parameter passed to submit() does not exceed available cluster resources. If set to <code>None</code>, it will default to the total number of vCPUs in the ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>cancel_futures</code> <p>Select which futures will be cancelled when exiting this context manager. <code>Pending</code> is the default, which will cancel all pending futures, but not running ones, as done by concurrent.futures.ProcessPoolExecutor. Additionally, <code>All</code> cancels all pending and running futures, and <code>None</code> doesn't cancel any. See CancellationPolicy</p> <p> TYPE: <code>Union[CancellationPolicy, bool]</code> DEFAULT: <code>ALL</code> </p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": None},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef __init__(\n    self,\n    max_workers: Optional[int] = None,\n    *,\n    config: Optional[ParallelConfig] = None,\n    cancel_futures: Union[CancellationPolicy, bool] = CancellationPolicy.ALL,\n):\n    if max_workers is not None:\n        if max_workers &lt;= 0:\n            raise ValueError(\"max_workers must be greater than 0\")\n        max_workers = max_workers\n\n    if isinstance(cancel_futures, CancellationPolicy):\n        self._cancel_futures = cancel_futures\n    else:\n        self._cancel_futures = (\n            CancellationPolicy.PENDING\n            if cancel_futures\n            else CancellationPolicy.NONE\n        )\n\n    if not ray.is_initialized():\n        raise RuntimeError(\n            \"Starting from v0.9.0, ray is no longer automatically initialized. \"\n            \"Please use `ray.init()` with the desired configuration \"\n            \"before using this class.\"\n        )\n\n    self._max_workers = max_workers\n    if self._max_workers is None:\n        self._max_workers = int(ray._private.state.cluster_resources()[\"CPU\"])\n\n    self._shutdown = False\n    self._shutdown_lock = threading.Lock()\n    self._queue_lock = threading.Lock()\n    self._work_queue: \"queue.Queue[Optional[_WorkItem]]\" = queue.Queue(\n        maxsize=self._max_workers\n    )\n    self._pending_queue: \"queue.SimpleQueue[Optional[_WorkItem]]\" = (\n        queue.SimpleQueue()\n    )\n\n    # Work Item Manager Thread\n    self._work_item_manager_thread: Optional[_WorkItemManagerThread] = None\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.submit","title":"submit","text":"<pre><code>submit(fn: Callable[..., T], *args, **kwargs) -&gt; Future[T]\n</code></pre> <p>Submits a callable to be executed with the given arguments.</p> <p>Schedules the callable to be executed as fn(*args, **kwargs) and returns a Future instance representing the execution of the callable.</p> PARAMETER  DESCRIPTION <code>fn</code> <p>Callable.</p> <p> TYPE: <code>Callable[..., T]</code> </p> <code>args</code> <p>Positional arguments that will be passed to <code>fn</code>.</p> <p> DEFAULT: <code>()</code> </p> <code>kwargs</code> <p>Keyword arguments that will be passed to <code>fn</code>. It can also optionally contain options for the ray remote function as a dictionary as the keyword argument <code>remote_function_options</code>.</p> <p> DEFAULT: <code>{}</code> </p> <p>Returns:     A Future representing the given call.</p> RAISES DESCRIPTION <code>RuntimeError</code> <p>If a task is submitted after the executor has been shut down.</p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def submit(self, fn: Callable[..., T], *args, **kwargs) -&gt; \"Future[T]\":\n    r\"\"\"Submits a callable to be executed with the given arguments.\n\n    Schedules the callable to be executed as fn(\\*args, \\**kwargs)\n    and returns a Future instance representing the execution of the callable.\n\n    Args:\n        fn: Callable.\n        args: Positional arguments that will be passed to `fn`.\n        kwargs: Keyword arguments that will be passed to `fn`.\n            It can also optionally contain options for the ray remote function\n            as a dictionary as the keyword argument `remote_function_options`.\n    Returns:\n        A Future representing the given call.\n\n    Raises:\n        RuntimeError: If a task is submitted after the executor has been shut down.\n    \"\"\"\n    with self._shutdown_lock:\n        logger.debug(\"executor acquired shutdown lock\")\n        if self._shutdown:\n            raise RuntimeError(\"cannot schedule new futures after shutdown\")\n\n        logging.debug(\"Creating future and putting work item in work queue\")\n        future: \"Future[T]\" = Future()\n        remote_function_options = kwargs.pop(\"remote_function_options\", None)\n        w = _WorkItem(\n            future,\n            fn,\n            args,\n            kwargs,\n            remote_function_options=remote_function_options,\n        )\n        self._put_work_item_in_queue(w)\n        # We delay starting the thread until the first call to submit\n        self._start_work_item_manager_thread()\n        return future\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.shutdown","title":"shutdown","text":"<pre><code>shutdown(wait: bool = True, *, cancel_futures: Optional[bool] = None) -&gt; None\n</code></pre> <p>Clean up the resources associated with the Executor.</p> <p>This method tries to mimic the behaviour of Executor.shutdown while allowing one more value for <code>cancel_futures</code> which instructs it to use the CancellationPolicy defined upon construction.</p> PARAMETER  DESCRIPTION <code>wait</code> <p>Whether to wait for pending futures to finish.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>cancel_futures</code> <p>Overrides the executor's default policy for cancelling futures on exit. If <code>True</code>, all pending futures are cancelled, and if <code>False</code>, no futures are cancelled. If <code>None</code> (default), the executor's policy set at initialization is used.</p> <p> TYPE: <code>Optional[bool]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def shutdown(\n    self, wait: bool = True, *, cancel_futures: Optional[bool] = None\n) -&gt; None:\n    \"\"\"Clean up the resources associated with the Executor.\n\n    This method tries to mimic the behaviour of\n    [Executor.shutdown][concurrent.futures.Executor.shutdown]\n    while allowing one more value for ``cancel_futures`` which instructs it\n    to use the [CancellationPolicy][pydvl.parallel.backend.CancellationPolicy]\n    defined upon construction.\n\n    Args:\n        wait: Whether to wait for pending futures to finish.\n        cancel_futures: Overrides the executor's default policy for\n            cancelling futures on exit. If ``True``, all pending futures are\n            cancelled, and if ``False``, no futures are cancelled. If ``None``\n            (default), the executor's policy set at initialization is used.\n    \"\"\"\n    logger.debug(\"executor shutting down\")\n    with self._shutdown_lock:\n        logger.debug(\"executor acquired shutdown lock\")\n        self._shutdown = True\n        self._cancel_futures = {\n            None: self._cancel_futures,\n            True: CancellationPolicy.PENDING,\n            False: CancellationPolicy.NONE,\n        }[cancel_futures]\n\n    if wait:\n        logger.debug(\"executor waiting for futures to finish\")\n        if self._work_item_manager_thread is not None:\n            # Putting None in the queue to signal\n            # to work item manager thread that we are shutting down\n            self._put_work_item_in_queue(None)\n            logger.debug(\n                \"executor waiting for work item manager thread to terminate\"\n            )\n            self._work_item_manager_thread.join()\n        # To reduce the risk of opening too many files, remove references to\n        # objects that use file descriptors.\n        self._work_item_manager_thread = None\n        del self._work_queue\n        del self._pending_queue\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.__exit__","title":"__exit__","text":"<pre><code>__exit__(exc_type, exc_val, exc_tb)\n</code></pre> <p>Exit the runtime context related to the RayExecutor object.</p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def __exit__(self, exc_type, exc_val, exc_tb):\n    \"\"\"Exit the runtime context related to the RayExecutor object.\"\"\"\n    self.shutdown()\n    return False\n</code></pre>"},{"location":"api/pydvl/reporting/","title":"Reporting","text":""},{"location":"api/pydvl/reporting/#pydvl.reporting","title":"pydvl.reporting","text":""},{"location":"api/pydvl/reporting/plots/","title":"Plots","text":""},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots","title":"pydvl.reporting.plots","text":""},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.shaded_mean_std","title":"shaded_mean_std","text":"<pre><code>shaded_mean_std(\n    data: ndarray,\n    abscissa: Optional[Sequence[Any]] = None,\n    num_std: float = 1.0,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    title: Optional[str] = None,\n    xlabel: Optional[str] = None,\n    ylabel: Optional[str] = None,\n    ax: Optional[Axes] = None,\n    **kwargs\n) -&gt; Axes\n</code></pre> <p>The usual mean \\(\\pm\\) std deviation plot to aggregate runs of experiments.</p> <p>Deprecation notice</p> <p>This function is bogus and will be removed in the future in favour of properly computed confidence intervals.</p> PARAMETER  DESCRIPTION <code>data</code> <p>axis 0 is to be aggregated on (e.g. runs) and axis 1 is the data for each run.</p> <p> TYPE: <code>ndarray</code> </p> <code>abscissa</code> <p>values for the x-axis. Leave empty to use increasing integers.</p> <p> TYPE: <code>Optional[Sequence[Any]]</code> DEFAULT: <code>None</code> </p> <code>num_std</code> <p>number of standard deviations to shade around the mean.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>mean_color</code> <p>color for the mean</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'dodgerblue'</code> </p> <code>shade_color</code> <p>color for the shaded region</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'lightblue'</code> </p> <code>title</code> <p>Title text. To use mathematics, use LaTeX notation.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>xlabel</code> <p>Text for the horizontal axis.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ylabel</code> <p>Text for the vertical axis</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ax</code> <p>If passed, axes object into which to insert the figure. Otherwise, a new figure is created and returned</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>these are forwarded to the ax.plot() call for the mean.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Axes</code> <p>The axes used (or created)</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>@deprecated(target=None, deprecated_in=\"0.7.1\", remove_in=\"0.9.0\")\ndef shaded_mean_std(\n    data: np.ndarray,\n    abscissa: Optional[Sequence[Any]] = None,\n    num_std: float = 1.0,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    title: Optional[str] = None,\n    xlabel: Optional[str] = None,\n    ylabel: Optional[str] = None,\n    ax: Optional[Axes] = None,\n    **kwargs,\n) -&gt; Axes:\n    r\"\"\"The usual mean \\(\\pm\\) std deviation plot to aggregate runs of\n    experiments.\n\n    !!! warning \"Deprecation notice\"\n        This function is bogus and will be removed in the future in favour of\n        properly computed confidence intervals.\n\n    Args:\n        data: axis 0 is to be aggregated on (e.g. runs) and axis 1 is the\n            data for each run.\n        abscissa: values for the x-axis. Leave empty to use increasing integers.\n        num_std: number of standard deviations to shade around the mean.\n        mean_color: color for the mean\n        shade_color: color for the shaded region\n        title: Title text. To use mathematics, use LaTeX notation.\n        xlabel: Text for the horizontal axis.\n        ylabel: Text for the vertical axis\n        ax: If passed, axes object into which to insert the figure. Otherwise,\n            a new figure is created and returned\n        kwargs: these are forwarded to the ax.plot() call for the mean.\n\n    Returns:\n        The axes used (or created)\n    \"\"\"\n    assert len(data.shape) == 2\n    mean = data.mean(axis=0)\n    std = num_std * data.std(axis=0)\n\n    if ax is None:\n        fig, ax = plt.subplots()\n    if abscissa is None:\n        abscissa = list(range(data.shape[1]))\n\n    ax.fill_between(abscissa, mean - std, mean + std, alpha=0.3, color=shade_color)\n    ax.plot(abscissa, mean, color=mean_color, **kwargs)\n\n    ax.set_title(title)\n    ax.set_xlabel(xlabel)\n    ax.set_ylabel(ylabel)\n\n    return ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_ci_array","title":"plot_ci_array","text":"<pre><code>plot_ci_array(\n    data: NDArray,\n    level: float,\n    type: Literal[\"normal\", \"t\", \"auto\"] = \"normal\",\n    abscissa: Optional[Sequence[str]] = None,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    ax: Optional[Axes] = None,\n    **kwargs\n) -&gt; Axes\n</code></pre> <p>Plot values and a confidence interval from a 2D array.</p> <p>Supported intervals are based on the normal and the t distributions.</p> PARAMETER  DESCRIPTION <code>data</code> <p>A 2D array with M different values for each of the N indices.</p> <p> TYPE: <code>NDArray</code> </p> <code>level</code> <p>The confidence level.</p> <p> TYPE: <code>float</code> </p> <code>type</code> <p>The type of confidence interval to use.</p> <p> TYPE: <code>Literal['normal', 't', 'auto']</code> DEFAULT: <code>'normal'</code> </p> <code>abscissa</code> <p>The values for the x-axis. Leave empty to use increasing integers.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>mean_color</code> <p>The color of the mean line.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'dodgerblue'</code> </p> <code>shade_color</code> <p>The color of the confidence interval.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'lightblue'</code> </p> <code>ax</code> <p>If passed, axes object into which to insert the figure. Otherwise, a new figure is created and the axes returned.</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>**kwargs</code> <p>Additional arguments to pass to the plot function.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Axes</code> <p>The matplotlib axes.</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_ci_array(\n    data: NDArray,\n    level: float,\n    type: Literal[\"normal\", \"t\", \"auto\"] = \"normal\",\n    abscissa: Optional[Sequence[str]] = None,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    ax: Optional[plt.Axes] = None,\n    **kwargs,\n) -&gt; plt.Axes:\n    \"\"\"Plot values and a confidence interval from a 2D array.\n\n    Supported intervals are based on the normal and the t distributions.\n\n    Args:\n        data: A 2D array with M different values for each of the N indices.\n        level: The confidence level.\n        type: The type of confidence interval to use.\n        abscissa: The values for the x-axis. Leave empty to use increasing\n            integers.\n        mean_color: The color of the mean line.\n        shade_color: The color of the confidence interval.\n        ax: If passed, axes object into which to insert the figure. Otherwise,\n            a new figure is created and the axes returned.\n        **kwargs: Additional arguments to pass to the plot function.\n\n    Returns:\n        The matplotlib axes.\n    \"\"\"\n\n    m, n = data.shape\n\n    means = np.mean(data, axis=0)\n    variances = np.var(data, axis=0, ddof=1)\n\n    dummy = ValuationResult[np.int_, np.object_](\n        algorithm=\"dummy\",\n        values=means,\n        variances=variances,\n        counts=np.ones_like(means, dtype=np.int_) * m,\n        indices=np.arange(n),\n        data_names=np.array(abscissa, dtype=str)\n        if abscissa is not None\n        else np.arange(n, dtype=str),\n    )\n\n    return plot_ci_values(\n        dummy,\n        level=level,\n        type=type,\n        mean_color=mean_color,\n        shade_color=shade_color,\n        ax=ax,\n        **kwargs,\n    )\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_ci_values","title":"plot_ci_values","text":"<pre><code>plot_ci_values(\n    values: ValuationResult,\n    level: float,\n    type: Literal[\"normal\", \"t\", \"auto\"] = \"auto\",\n    abscissa: Optional[Sequence[str]] = None,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    ax: Optional[Axes] = None,\n    **kwargs\n)\n</code></pre> <p>Plot values and a confidence interval.</p> <p>Uses <code>values.data_names</code> for the x-axis.</p> <p>Supported intervals are based on the normal and the t distributions.</p> PARAMETER  DESCRIPTION <code>values</code> <p>The valuation result.</p> <p> TYPE: <code>ValuationResult</code> </p> <code>level</code> <p>The confidence level.</p> <p> TYPE: <code>float</code> </p> <code>type</code> <p>The type of confidence interval to use. If \"auto\", uses \"norm\" if the minimum number of updates for all indices is greater than 30, otherwise uses \"t\".</p> <p> TYPE: <code>Literal['normal', 't', 'auto']</code> DEFAULT: <code>'auto'</code> </p> <code>abscissa</code> <p>The values for the x-axis. Leave empty to use increasing integers.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>mean_color</code> <p>The color of the mean line.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'dodgerblue'</code> </p> <code>shade_color</code> <p>The color of the confidence interval.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'lightblue'</code> </p> <code>ax</code> <p>If passed, axes object into which to insert the figure. Otherwise, a new figure is created and the axes returned.</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>**kwargs</code> <p>Additional arguments to pass to the plot function.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <p>The matplotlib axes.</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_ci_values(\n    values: ValuationResult,\n    level: float,\n    type: Literal[\"normal\", \"t\", \"auto\"] = \"auto\",\n    abscissa: Optional[Sequence[str]] = None,\n    mean_color: Optional[str] = \"dodgerblue\",\n    shade_color: Optional[str] = \"lightblue\",\n    ax: Optional[plt.Axes] = None,\n    **kwargs,\n):\n    \"\"\"Plot values and a confidence interval.\n\n    Uses `values.data_names` for the x-axis.\n\n    Supported intervals are based on the normal and the t distributions.\n\n    Args:\n        values: The valuation result.\n        level: The confidence level.\n        type: The type of confidence interval to use. If \"auto\", uses \"norm\" if\n            the minimum number of updates for all indices is greater than 30,\n            otherwise uses \"t\".\n        abscissa: The values for the x-axis. Leave empty to use increasing\n            integers.\n        mean_color: The color of the mean line.\n        shade_color: The color of the confidence interval.\n        ax: If passed, axes object into which to insert the figure. Otherwise,\n            a new figure is created and the axes returned.\n        **kwargs: Additional arguments to pass to the plot function.\n\n    Returns:\n        The matplotlib axes.\n    \"\"\"\n\n    ppfs = {\n        \"normal\": norm.ppf,\n        \"t\": partial(t.ppf, df=values.counts - 1),\n        \"auto\": norm.ppf\n        if np.min(values.counts) &gt; 30\n        else partial(t.ppf, df=values.counts - 1),\n    }\n\n    try:\n        score = ppfs[type](1 - level / 2)\n    except KeyError:\n        raise ValueError(\n            f\"Unknown confidence interval type requested: {type}.\"\n        ) from None\n\n    if abscissa is None:\n        abscissa = [str(i) for i, _ in enumerate(values)]\n    bound = score * values.stderr\n\n    if ax is None:\n        fig, ax = plt.subplots()\n\n    ax.fill_between(\n        abscissa,\n        values.values - bound,\n        values.values + bound,\n        alpha=0.3,\n        color=shade_color,\n    )\n    ax.plot(abscissa, values.values, color=mean_color, **kwargs)\n    return ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.spearman_correlation","title":"spearman_correlation","text":"<pre><code>spearman_correlation(vv: List[OrderedDict], num_values: int, pvalue: float)\n</code></pre> <p>Simple matrix plots with spearman correlation for each pair in vv.</p> PARAMETER  DESCRIPTION <code>vv</code> <p>list of OrderedDicts with index: value. Spearman correlation is computed for the keys.</p> <p> TYPE: <code>List[OrderedDict]</code> </p> <code>num_values</code> <p>Use only these many values from the data (from the start of the OrderedDicts)</p> <p> TYPE: <code>int</code> </p> <code>pvalue</code> <p>correlation coefficients for which the p-value is below the threshold <code>pvalue/len(vv)</code> will be discarded.</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def spearman_correlation(vv: List[OrderedDict], num_values: int, pvalue: float):\n    \"\"\"Simple matrix plots with spearman correlation for each pair in vv.\n\n    Args:\n        vv: list of OrderedDicts with index: value. Spearman correlation\n            is computed for the keys.\n        num_values: Use only these many values from the data (from the start\n            of the OrderedDicts)\n        pvalue: correlation coefficients for which the p-value is below the\n            threshold `pvalue/len(vv)` will be discarded.\n    \"\"\"\n    r: np.ndarray = np.ndarray((len(vv), len(vv)))\n    p: np.ndarray = np.ndarray((len(vv), len(vv)))\n    for i, a in enumerate(vv):\n        for j, b in enumerate(vv):\n            from scipy.stats._stats_py import SpearmanrResult\n\n            spearman: SpearmanrResult = sp.stats.spearmanr(\n                list(a.keys())[:num_values], list(b.keys())[:num_values]\n            )\n            r[i][j] = (\n                spearman.correlation if spearman.pvalue &lt; pvalue / len(vv) else np.nan\n            )  # Bonferroni correction\n            p[i][j] = spearman.pvalue\n    fig, axs = plt.subplots(1, 2, figsize=(16, 7))\n    plot1 = axs[0].matshow(r, vmin=-1, vmax=1)\n    axs[0].set_title(f\"Spearman correlation (top {num_values} values)\")\n    axs[0].set_xlabel(\"Runs\")\n    axs[0].set_ylabel(\"Runs\")\n    fig.colorbar(plot1, ax=axs[0])\n    plot2 = axs[1].matshow(p, vmin=0, vmax=1)\n    axs[1].set_title(\"p-value\")\n    axs[1].set_xlabel(\"Runs\")\n    axs[1].set_ylabel(\"Runs\")\n    fig.colorbar(plot2, ax=axs[1])\n\n    return fig\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_shapley","title":"plot_shapley","text":"<pre><code>plot_shapley(\n    df: DataFrame,\n    *,\n    level: float = 0.05,\n    ax: Optional[Axes] = None,\n    title: Optional[str] = None,\n    xlabel: Optional[str] = None,\n    ylabel: Optional[str] = None,\n    prefix: Optional[str] = \"data_value\"\n) -&gt; Axes\n</code></pre> <p>Plots the shapley values, as returned from compute_shapley_values, with error bars corresponding to an \\(\\alpha\\)-level Normal confidence interval.</p> PARAMETER  DESCRIPTION <code>df</code> <p>dataframe with the shapley values</p> <p> TYPE: <code>DataFrame</code> </p> <code>level</code> <p>confidence level for the error bars</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.05</code> </p> <code>ax</code> <p>axes to plot on or None if a new subplots should be created</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>title</code> <p>string, title of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>xlabel</code> <p>string, x label of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ylabel</code> <p>string, y label of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Axes</code> <p>The axes created or used</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_shapley(\n    df: pd.DataFrame,\n    *,\n    level: float = 0.05,\n    ax: Optional[plt.Axes] = None,\n    title: Optional[str] = None,\n    xlabel: Optional[str] = None,\n    ylabel: Optional[str] = None,\n    prefix: Optional[str] = \"data_value\",\n) -&gt; plt.Axes:\n    r\"\"\"Plots the shapley values, as returned from\n    [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values],\n    with error bars corresponding to an $\\alpha$-level Normal confidence\n    interval.\n\n    Args:\n        df: dataframe with the shapley values\n        level: confidence level for the error bars\n        ax: axes to plot on or None if a new subplots should be created\n        title: string, title of the plot\n        xlabel: string, x label of the plot\n        ylabel: string, y label of the plot\n\n    Returns:\n        The axes created or used\n    \"\"\"\n    if ax is None:\n        _, ax = plt.subplots()\n\n    yerr = norm.ppf(1 - level / 2) * df[f\"{prefix}_stderr\"]\n\n    ax.errorbar(x=df.index, y=df[prefix], yerr=yerr, fmt=\"o\", capsize=6)\n    ax.set_xlabel(xlabel)\n    ax.set_ylabel(ylabel)\n    ax.set_title(title)\n    plt.xticks(rotation=60)\n    return ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_influence_distribution","title":"plot_influence_distribution","text":"<pre><code>plot_influence_distribution(\n    influences: NDArray[float_], index: int, title_extra: str = \"\"\n) -&gt; Axes\n</code></pre> <p>Plots the histogram of the influence that all samples in the training set have over a single sample index.</p> PARAMETER  DESCRIPTION <code>influences</code> <p>array of influences (training samples x test samples)</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>index</code> <p>Index of the test sample for which the influences   will be plotted.</p> <p> TYPE: <code>int</code> </p> <code>title_extra</code> <p>Additional text that will be appended to the title.</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_influence_distribution(\n    influences: NDArray[np.float_], index: int, title_extra: str = \"\"\n) -&gt; plt.Axes:\n    \"\"\"Plots the histogram of the influence that all samples in the training set\n    have over a single sample index.\n\n    Args:\n       influences: array of influences (training samples x test samples)\n       index: Index of the test sample for which the influences\n            will be plotted.\n       title_extra: Additional text that will be appended to the title.\n    \"\"\"\n    _, ax = plt.subplots()\n    ax.hist(influences[:, index], alpha=0.7)\n    ax.set_xlabel(\"Influence values\")\n    ax.set_ylabel(\"Number of samples\")\n    ax.set_title(f\"Distribution of influences {title_extra}\")\n    return ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_influence_distribution_by_label","title":"plot_influence_distribution_by_label","text":"<pre><code>plot_influence_distribution_by_label(\n    influences: NDArray[float_], labels: NDArray[float_], title_extra: str = \"\"\n)\n</code></pre> <p>Plots the histogram of the influence that all samples in the training set have over a single sample index, separated by labels.</p> PARAMETER  DESCRIPTION <code>influences</code> <p>array of influences (training samples x test samples)</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>labels</code> <p>labels for the training set.</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>title_extra</code> <p>Additional text that will be appended to the title.</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_influence_distribution_by_label(\n    influences: NDArray[np.float_], labels: NDArray[np.float_], title_extra: str = \"\"\n):\n    \"\"\"Plots the histogram of the influence that all samples in the training set\n    have over a single sample index, separated by labels.\n\n    Args:\n       influences: array of influences (training samples x test samples)\n       labels: labels for the training set.\n       title_extra: Additional text that will be appended to the title.\n    \"\"\"\n    _, ax = plt.subplots()\n    unique_labels = np.unique(labels)\n    for label in unique_labels:\n        ax.hist(influences[labels == label], label=label, alpha=0.7)\n    ax.set_xlabel(\"Influence values\")\n    ax.set_ylabel(\"Number of samples\")\n    ax.set_title(f\"Distribution of influences {title_extra}\")\n    ax.legend()\n    plt.show()\n</code></pre>"},{"location":"api/pydvl/reporting/scores/","title":"Scores","text":""},{"location":"api/pydvl/reporting/scores/#pydvl.reporting.scores","title":"pydvl.reporting.scores","text":""},{"location":"api/pydvl/reporting/scores/#pydvl.reporting.scores.compute_removal_score","title":"compute_removal_score","text":"<pre><code>compute_removal_score(\n    u: Utility,\n    values: ValuationResult,\n    percentages: Union[NDArray[float_], Iterable[float]],\n    *,\n    remove_best: bool = False,\n    progress: bool = False\n) -&gt; Dict[float, float]\n</code></pre> <p>Fits model and computes score on the test set after incrementally removing a percentage of data points from the training set, based on their values.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>values</code> <p>Data values of data instances in the training set.</p> <p> TYPE: <code>ValuationResult</code> </p> <code>percentages</code> <p>Sequence of removal percentages.</p> <p> TYPE: <code>Union[NDArray[float_], Iterable[float]]</code> </p> <code>remove_best</code> <p>If True, removes data points in order of decreasing valuation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>progress</code> <p>If True, display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Dict[float, float]</code> <p>Dictionary that maps the percentages to their respective scores.</p> Source code in <code>src/pydvl/reporting/scores.py</code> <pre><code>def compute_removal_score(\n    u: Utility,\n    values: ValuationResult,\n    percentages: Union[NDArray[np.float_], Iterable[float]],\n    *,\n    remove_best: bool = False,\n    progress: bool = False,\n) -&gt; Dict[float, float]:\n    r\"\"\"Fits model and computes score on the test set after incrementally removing\n    a percentage of data points from the training set, based on their values.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        values: Data values of data instances in the training set.\n        percentages: Sequence of removal percentages.\n        remove_best: If True, removes data points in order of decreasing valuation.\n        progress: If True, display a progress bar.\n\n    Returns:\n        Dictionary that maps the percentages to their respective scores.\n    \"\"\"\n    # Sanity checks\n    if np.any([x &gt;= 1.0 or x &lt; 0.0 for x in percentages]):\n        raise ValueError(\"All percentages should be in the range [0.0, 1.0)\")\n\n    if len(values) != len(u.data.indices):\n        raise ValueError(\n            f\"The number of values, {len(values) }, should be equal to the number of data indices, {len(u.data.indices)}\"\n        )\n\n    scores = {}\n\n    # We sort in descending order if we want to remove the best values\n    values.sort(reverse=remove_best)\n\n    for pct in tqdm(percentages, disable=not progress, desc=\"Removal Scores\"):\n        n_removal = int(pct * len(u.data))\n        indices = values.indices[n_removal:]\n        score = u(indices)\n        scores[pct] = score\n    return scores\n</code></pre>"},{"location":"api/pydvl/utils/","title":"Utils","text":""},{"location":"api/pydvl/utils/#pydvl.utils","title":"pydvl.utils","text":""},{"location":"api/pydvl/utils/config/","title":"Config","text":""},{"location":"api/pydvl/utils/config/#pydvl.utils.config","title":"pydvl.utils.config","text":""},{"location":"api/pydvl/utils/config/#pydvl.utils.config.ParallelConfig","title":"ParallelConfig  <code>dataclass</code>","text":"<pre><code>ParallelConfig(\n    backend: Literal[\"joblib\", \"ray\"] = \"joblib\",\n    address: Optional[Union[str, Tuple[str, int]]] = None,\n    n_cpus_local: Optional[int] = None,\n    logging_level: Optional[int] = None,\n    wait_timeout: float = 1.0,\n)\n</code></pre> <p>Configuration for parallel computation backend.</p> PARAMETER  DESCRIPTION <code>backend</code> <p>Type of backend to use. Defaults to 'joblib'</p> <p> TYPE: <code>Literal['joblib', 'ray']</code> DEFAULT: <code>'joblib'</code> </p> <code>address</code> <p>(DEPRECATED) Address of existing remote or local cluster to use.</p> <p> TYPE: <code>Optional[Union[str, Tuple[str, int]]]</code> DEFAULT: <code>None</code> </p> <code>n_cpus_local</code> <p>(DEPRECATED) Number of CPUs to use when creating a local ray cluster. This has no effect when using an existing ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>logging_level</code> <p>(DEPRECATED) Logging level for the parallel backend's worker.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>wait_timeout</code> <p>(DEPRECATED) Timeout in seconds for waiting on futures.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p>"},{"location":"api/pydvl/utils/config/#pydvl.utils.config.CachedFuncConfig","title":"CachedFuncConfig  <code>dataclass</code>","text":"<pre><code>CachedFuncConfig(\n    hash_prefix: Optional[str] = None,\n    ignore_args: Collection[str] = list(),\n    time_threshold: float = 0.3,\n    allow_repeated_evaluations: bool = False,\n    rtol_stderr: float = 0.1,\n    min_repetitions: int = 3,\n)\n</code></pre> <p>Configuration for cached functions and methods, providing memoization of function calls.</p> <p>Instances of this class are typically used as arguments for the construction of a Utility.</p> PARAMETER  DESCRIPTION <code>hash_prefix</code> <p>Optional string prefix that be prepended to the cache key. This can be provided in order to guarantee cache reuse across runs.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ignore_args</code> <p>Do not take these keyword arguments into account when hashing the wrapped function for usage as key. This allows sharing the cache among different jobs for the same experiment run if the callable happens to have \"nuisance\" parameters like <code>job_id</code> which do not affect the result of the computation.</p> <p> TYPE: <code>Collection[str]</code> DEFAULT: <code>list()</code> </p> <code>time_threshold</code> <p>Computations taking less time than this many seconds are not cached. A value of 0 means that it will always cache results.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.3</code> </p> <code>allow_repeated_evaluations</code> <p>If <code>True</code>, repeated calls to a function with the same arguments will be allowed and outputs averaged until the running standard deviation of the mean stabilizes below <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>rtol_stderr</code> <p>relative tolerance for repeated evaluations. More precisely, memcached() will stop evaluating the function once the standard deviation of the mean is smaller than <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.1</code> </p> <code>min_repetitions</code> <p>minimum number of times that a function evaluation on the same arguments is repeated before returning cached values. Useful for stochastic functions only. If the model training is very noisy, set this number to higher values to reduce variance.</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p>"},{"location":"api/pydvl/utils/dataset/","title":"Dataset","text":""},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset","title":"pydvl.utils.dataset","text":"<p>This module contains convenience classes to handle data and groups thereof.</p> <p>Shapley and Least Core value computations require evaluation of a scoring function (the utility). This is typically the performance of the model on a test set (as an approximation to its true expected performance). It is therefore convenient to keep both the training data and the test data together to be passed around to methods in shapley and least_core. This is done with Dataset.</p> <p>This abstraction layer also seamlessly grouping data points together if one is interested in computing their value as a group, see GroupedDataset.</p> <p>Objects of both types are used to construct a Utility object.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset","title":"Dataset","text":"<pre><code>Dataset(\n    x_train: Union[NDArray, DataFrame],\n    y_train: Union[NDArray, DataFrame],\n    x_test: Union[NDArray, DataFrame],\n    y_test: Union[NDArray, DataFrame],\n    feature_names: Optional[Sequence[str]] = None,\n    target_names: Optional[Sequence[str]] = None,\n    data_names: Optional[Sequence[str]] = None,\n    description: Optional[str] = None,\n    is_multi_output: bool = False,\n)\n</code></pre> <p>A convenience class to handle datasets.</p> <p>It holds a dataset, split into training and test data, together with several labels on feature names, data point names and a description.</p> PARAMETER  DESCRIPTION <code>x_train</code> <p>training data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>y_train</code> <p>labels for training data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>x_test</code> <p>test data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>y_test</code> <p>labels for test data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>feature_names</code> <p>name of the features of input data</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>target_names</code> <p>names of the features of target data</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>names assigned to data points. For example, if the dataset is a time series, each entry can be a timestamp which can be referenced directly instead of using a row number.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>description</code> <p>A textual description of the dataset.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>is_multi_output</code> <p>set to <code>False</code> if labels are scalars, or to <code>True</code> if they are vectors of dimension &gt; 1.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def __init__(\n    self,\n    x_train: Union[NDArray, pd.DataFrame],\n    y_train: Union[NDArray, pd.DataFrame],\n    x_test: Union[NDArray, pd.DataFrame],\n    y_test: Union[NDArray, pd.DataFrame],\n    feature_names: Optional[Sequence[str]] = None,\n    target_names: Optional[Sequence[str]] = None,\n    data_names: Optional[Sequence[str]] = None,\n    description: Optional[str] = None,\n    # FIXME: use same parameter name as in check_X_y()\n    is_multi_output: bool = False,\n):\n    \"\"\"Constructs a Dataset from data and labels.\n\n    Args:\n        x_train: training data\n        y_train: labels for training data\n        x_test: test data\n        y_test: labels for test data\n        feature_names: name of the features of input data\n        target_names: names of the features of target data\n        data_names: names assigned to data points.\n            For example, if the dataset is a time series, each entry can be a\n            timestamp which can be referenced directly instead of using a row\n            number.\n        description: A textual description of the dataset.\n        is_multi_output: set to `False` if labels are scalars, or to\n            `True` if they are vectors of dimension &gt; 1.\n    \"\"\"\n    self.x_train, self.y_train = check_X_y(\n        x_train, y_train, multi_output=is_multi_output\n    )\n    self.x_test, self.y_test = check_X_y(\n        x_test, y_test, multi_output=is_multi_output\n    )\n\n    if x_train.shape[-1] != x_test.shape[-1]:\n        raise ValueError(\n            f\"Mismatching number of features: \"\n            f\"{x_train.shape[-1]} and {x_test.shape[-1]}\"\n        )\n    if x_train.shape[0] != y_train.shape[0]:\n        raise ValueError(\n            f\"Mismatching number of samples: \"\n            f\"{x_train.shape[-1]} and {x_test.shape[-1]}\"\n        )\n    if x_test.shape[0] != y_test.shape[0]:\n        raise ValueError(\n            f\"Mismatching number of samples: \"\n            f\"{x_test.shape[-1]} and {y_test.shape[-1]}\"\n        )\n\n    def make_names(s: str, a: np.ndarray) -&gt; List[str]:\n        n = a.shape[1] if len(a.shape) &gt; 1 else 1\n        return [f\"{s}{i:0{1 + int(np.log10(n))}d}\" for i in range(1, n + 1)]\n\n    self.feature_names = feature_names\n    self.target_names = target_names\n\n    if self.feature_names is None:\n        if isinstance(x_train, pd.DataFrame):\n            self.feature_names = x_train.columns.tolist()\n        else:\n            self.feature_names = make_names(\"x\", x_train)\n\n    if self.target_names is None:\n        if isinstance(y_train, pd.DataFrame):\n            self.target_names = y_train.columns.tolist()\n        else:\n            self.target_names = make_names(\"y\", y_train)\n\n    if len(self.x_train.shape) &gt; 1:\n        if (\n            len(self.feature_names) != self.x_train.shape[-1]\n            or len(self.feature_names) != self.x_test.shape[-1]\n        ):\n            raise ValueError(\"Mismatching number of features and names\")\n    if len(self.y_train.shape) &gt; 1:\n        if (\n            len(self.target_names) != self.y_train.shape[-1]\n            or len(self.target_names) != self.y_test.shape[-1]\n        ):\n            raise ValueError(\"Mismatching number of targets and names\")\n\n    self.description = description or \"No description\"\n    self._indices = np.arange(len(self.x_train), dtype=np.int_)\n    self._data_names = (\n        np.array(data_names, dtype=object)\n        if data_names is not None\n        else self._indices.astype(object)\n    )\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.indices","title":"indices  <code>property</code>","text":"<pre><code>indices: NDArray[int_]\n</code></pre> <p>Index of positions in data.x_train.</p> <p>Contiguous integers from 0 to len(Dataset).</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.data_names","title":"data_names  <code>property</code>","text":"<pre><code>data_names: NDArray[object_]\n</code></pre> <p>Names of each individual datapoint.</p> <p>Used for reporting Shapley values.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.dim","title":"dim  <code>property</code>","text":"<pre><code>dim: int\n</code></pre> <p>Returns the number of dimensions of a sample.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.get_training_data","title":"get_training_data","text":"<pre><code>get_training_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Given a set of indices, returns the training data that refer to those indices.</p> <p>This is used mainly by Utility to retrieve subsets of the data from indices. It is typically not needed in algorithms.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices that will be used to select points from the training data. If <code>None</code>, the entire training data will be returned.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>If <code>indices</code> is not <code>None</code>, the selected x and y arrays from the training data. Otherwise, the entire dataset.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_training_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Given a set of indices, returns the training data that refer to those\n    indices.\n\n    This is used mainly by [Utility][pydvl.utils.utility.Utility] to retrieve\n    subsets of the data from indices. It is typically **not needed in\n    algorithms**.\n\n    Args:\n        indices: Optional indices that will be used to select points from\n            the training data. If `None`, the entire training data will be\n            returned.\n\n    Returns:\n        If `indices` is not `None`, the selected x and y arrays from the\n            training data. Otherwise, the entire dataset.\n    \"\"\"\n    if indices is None:\n        return self.x_train, self.y_train\n    x = self.x_train[indices]\n    y = self.y_train[indices]\n    return x, y\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.get_test_data","title":"get_test_data","text":"<pre><code>get_test_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Returns the entire test set regardless of the passed indices.</p> <p>The passed indices will not be used because for data valuation we generally want to score the trained model on the entire test data.</p> <p>Additionally, the way this method is used in the Utility class, the passed indices will be those of the training data and would not work on the test data.</p> <p>There may be cases where it is desired to use parts of the test data. In those cases, it is recommended to inherit from Dataset and override get_test_data().</p> <p>For example, the following snippet shows how one could go about mapping the training data indices into test data indices inside get_test_data():</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n...    def get_test_data(self, indices=None):\n...        if indices is None:\n...            return self.x_test, self.y_test\n...        fraction = len(list(indices)) / len(self)\n...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n...        mapped_indices = np.unique(mapped_indices.astype(int))\n...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n...\n&gt;&gt;&gt; X = np.random.rand(100, 10)\n&gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n&gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n&gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n&gt;&gt;&gt; _ = dataset.get_training_data(indices)\n&gt;&gt;&gt; _ = dataset.get_test_data(indices)\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices into the test data. This argument is unused left for compatibility with get_training_data().</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>The entire test data.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_test_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Returns the entire test set regardless of the passed indices.\n\n    The passed indices will not be used because for data valuation\n    we generally want to score the trained model on the entire test data.\n\n    Additionally, the way this method is used in the\n    [Utility][pydvl.utils.utility.Utility] class, the passed indices will\n    be those of the training data and would not work on the test data.\n\n    There may be cases where it is desired to use parts of the test data.\n    In those cases, it is recommended to inherit from\n    [Dataset][pydvl.utils.dataset.Dataset] and override\n    [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data].\n\n    For example, the following snippet shows how one could go about\n    mapping the training data indices into test data indices\n    inside [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data]:\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n        ...    def get_test_data(self, indices=None):\n        ...        if indices is None:\n        ...            return self.x_test, self.y_test\n        ...        fraction = len(list(indices)) / len(self)\n        ...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n        ...        mapped_indices = np.unique(mapped_indices.astype(int))\n        ...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n        ...\n        &gt;&gt;&gt; X = np.random.rand(100, 10)\n        &gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n        &gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n        &gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n        &gt;&gt;&gt; _ = dataset.get_training_data(indices)\n        &gt;&gt;&gt; _ = dataset.get_test_data(indices)\n        ```\n\n    Args:\n        indices: Optional indices into the test data. This argument is\n            unused left for compatibility with\n            [get_training_data()][pydvl.utils.dataset.Dataset.get_training_data].\n\n    Returns:\n        The entire test data.\n    \"\"\"\n    return self.x_test, self.y_test\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.from_sklearn","title":"from_sklearn  <code>classmethod</code>","text":"<pre><code>from_sklearn(\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs\n) -&gt; Dataset\n</code></pre> <p>Constructs a Dataset object from a sklearn.utils.Bunch, as returned by the <code>load_*</code> functions in scikit-learn toy datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import load_boston\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n</code></pre> PARAMETER  DESCRIPTION <code>data</code> <p>scikit-learn Bunch object. The following attributes are supported:</p> <ul> <li><code>data</code>: covariates.</li> <li><code>target</code>: target variables (labels).</li> <li><code>feature_names</code> (optional): the feature names.</li> <li><code>target_names</code> (optional): the target names.</li> <li><code>DESCR</code> (optional): a description.</li> </ul> <p> TYPE: <code>Bunch</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the target variable as labels. Read more in scikit-learn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>is_multi_output</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the sklearn dataset</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_sklearn(\n    cls,\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs,\n) -&gt; \"Dataset\":\n    \"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from a\n    [sklearn.utils.Bunch][], as returned by the `load_*`\n    functions in [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html).\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import load_boston\n        &gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n        ```\n\n    Args:\n        data: scikit-learn Bunch object. The following attributes are supported:\n\n            - `data`: covariates.\n            - `target`: target variables (labels).\n            - `feature_names` (**optional**): the feature names.\n            - `target_names` (**optional**): the target names.\n            - `DESCR` (**optional**): a description.\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the target variable as labels. Read more in\n            [scikit-learn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `is_multi_output`.\n\n    Returns:\n        Object with the sklearn dataset\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\n    x_train, x_test, y_train, y_test = train_test_split(\n        data.data,\n        data.target,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=data.target if stratify_by_target else None,\n    )\n    return cls(\n        x_train,\n        y_train,\n        x_test,\n        y_test,\n        feature_names=data.get(\"feature_names\"),\n        target_names=data.get(\"target_names\"),\n        description=data.get(\"DESCR\"),\n        **kwargs,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.from_arrays","title":"from_arrays  <code>classmethod</code>","text":"<pre><code>from_arrays(\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs\n) -&gt; Dataset\n</code></pre> <p>Constructs a Dataset object from X and y numpy arrays  as returned by the <code>make_*</code> functions in sklearn generated datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import make_regression\n&gt;&gt;&gt; X, y = make_regression()\n&gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n</code></pre> PARAMETER  DESCRIPTION <code>X</code> <p>numpy array of shape (n_samples, n_features)</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>numpy array of shape (n_samples,)</p> <p> TYPE: <code>NDArray</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the y variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>feature_names</code> or <code>target_names</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the passed X and y arrays split across training and test sets.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_arrays(\n    cls,\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs,\n) -&gt; \"Dataset\":\n    \"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from X and y numpy arrays  as\n    returned by the `make_*` functions in [sklearn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import make_regression\n        &gt;&gt;&gt; X, y = make_regression()\n        &gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n        ```\n\n    Args:\n        X: numpy array of shape (n_samples, n_features)\n        y: numpy array of shape (n_samples,)\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified fashion,\n            using the y variable as labels. Read more in [sklearn's user\n            guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `feature_names`\n            or `target_names`.\n\n    Returns:\n        Object with the passed X and y arrays split across training and test sets.\n\n    !!! tip \"New in version 0.4.0\"\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\n    x_train, x_test, y_train, y_test = train_test_split(\n        X,\n        y,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=y if stratify_by_target else None,\n    )\n    return cls(x_train, y_train, x_test, y_test, **kwargs)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset","title":"GroupedDataset","text":"<pre><code>GroupedDataset(\n    x_train: NDArray,\n    y_train: NDArray,\n    x_test: NDArray,\n    y_test: NDArray,\n    data_groups: Sequence,\n    feature_names: Optional[Sequence[str]] = None,\n    target_names: Optional[Sequence[str]] = None,\n    group_names: Optional[Sequence[str]] = None,\n    description: Optional[str] = None,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>Dataset</code></p> <p>Used for calculating Shapley values of subsets of the data considered as logical units. For instance, one can group by value of a categorical feature, by bin into which a continuous feature falls, or by label.</p> PARAMETER  DESCRIPTION <code>x_train</code> <p>training data</p> <p> TYPE: <code>NDArray</code> </p> <code>y_train</code> <p>labels of training data</p> <p> TYPE: <code>NDArray</code> </p> <code>x_test</code> <p>test data</p> <p> TYPE: <code>NDArray</code> </p> <code>y_test</code> <p>labels of test data</p> <p> TYPE: <code>NDArray</code> </p> <code>data_groups</code> <p>Iterable of the same length as <code>x_train</code> containing a group label for each training data point. The label can be of any type, e.g. <code>str</code> or <code>int</code>. Data points with the same label will then be grouped by this object and considered as one for effects of valuation.</p> <p> TYPE: <code>Sequence</code> </p> <code>feature_names</code> <p>names of the covariates' features.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>target_names</code> <p>names of the labels or targets y</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>group_names</code> <p>names of the groups. If not provided, the labels from <code>data_groups</code> will be used.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>description</code> <p>A textual description of the dataset</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> <p>Changed in version 0.6.0</p> <p>Added <code>group_names</code> and forwarding of <code>kwargs</code></p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def __init__(\n    self,\n    x_train: NDArray,\n    y_train: NDArray,\n    x_test: NDArray,\n    y_test: NDArray,\n    data_groups: Sequence,\n    feature_names: Optional[Sequence[str]] = None,\n    target_names: Optional[Sequence[str]] = None,\n    group_names: Optional[Sequence[str]] = None,\n    description: Optional[str] = None,\n    **kwargs,\n):\n    \"\"\"Class for grouping datasets.\n\n    Used for calculating Shapley values of subsets of the data considered\n    as logical units. For instance, one can group by value of a categorical\n    feature, by bin into which a continuous feature falls, or by label.\n\n    Args:\n        x_train: training data\n        y_train: labels of training data\n        x_test: test data\n        y_test: labels of test data\n        data_groups: Iterable of the same length as `x_train` containing\n            a group label for each training data point. The label can be of any\n            type, e.g. `str` or `int`. Data points with the same label will\n            then be grouped by this object and considered as one for effects of\n            valuation.\n        feature_names: names of the covariates' features.\n        target_names: names of the labels or targets y\n        group_names: names of the groups. If not provided, the labels\n            from `data_groups` will be used.\n        description: A textual description of the dataset\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n\n    !!! tip \"Changed in version 0.6.0\"\n    Added `group_names` and forwarding of `kwargs`\n    \"\"\"\n    super().__init__(\n        x_train=x_train,\n        y_train=y_train,\n        x_test=x_test,\n        y_test=y_test,\n        feature_names=feature_names,\n        target_names=target_names,\n        description=description,\n        **kwargs,\n    )\n\n    if len(data_groups) != len(x_train):\n        raise ValueError(\n            f\"data_groups and x_train must have the same length.\"\n            f\"Instead got {len(data_groups)=} and {len(x_train)=}\"\n        )\n\n    self.groups: OrderedDict[Any, List[int]] = OrderedDict(\n        {k: [] for k in set(data_groups)}\n    )\n    for idx, group in enumerate(data_groups):\n        self.groups[group].append(idx)\n    self.group_items = list(self.groups.items())\n    self._indices = np.arange(len(self.groups.keys()))\n    self._data_names = (\n        np.array(group_names, dtype=object)\n        if group_names is not None\n        else np.array(list(self.groups.keys()), dtype=object)\n    )\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.dim","title":"dim  <code>property</code>","text":"<pre><code>dim: int\n</code></pre> <p>Returns the number of dimensions of a sample.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.indices","title":"indices  <code>property</code>","text":"<pre><code>indices\n</code></pre> <p>Indices of the groups.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.data_names","title":"data_names  <code>property</code>","text":"<pre><code>data_names\n</code></pre> <p>Names of the groups.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.get_test_data","title":"get_test_data","text":"<pre><code>get_test_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Returns the entire test set regardless of the passed indices.</p> <p>The passed indices will not be used because for data valuation we generally want to score the trained model on the entire test data.</p> <p>Additionally, the way this method is used in the Utility class, the passed indices will be those of the training data and would not work on the test data.</p> <p>There may be cases where it is desired to use parts of the test data. In those cases, it is recommended to inherit from Dataset and override get_test_data().</p> <p>For example, the following snippet shows how one could go about mapping the training data indices into test data indices inside get_test_data():</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n...    def get_test_data(self, indices=None):\n...        if indices is None:\n...            return self.x_test, self.y_test\n...        fraction = len(list(indices)) / len(self)\n...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n...        mapped_indices = np.unique(mapped_indices.astype(int))\n...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n...\n&gt;&gt;&gt; X = np.random.rand(100, 10)\n&gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n&gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n&gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n&gt;&gt;&gt; _ = dataset.get_training_data(indices)\n&gt;&gt;&gt; _ = dataset.get_test_data(indices)\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices into the test data. This argument is unused left for compatibility with get_training_data().</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>The entire test data.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_test_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Returns the entire test set regardless of the passed indices.\n\n    The passed indices will not be used because for data valuation\n    we generally want to score the trained model on the entire test data.\n\n    Additionally, the way this method is used in the\n    [Utility][pydvl.utils.utility.Utility] class, the passed indices will\n    be those of the training data and would not work on the test data.\n\n    There may be cases where it is desired to use parts of the test data.\n    In those cases, it is recommended to inherit from\n    [Dataset][pydvl.utils.dataset.Dataset] and override\n    [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data].\n\n    For example, the following snippet shows how one could go about\n    mapping the training data indices into test data indices\n    inside [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data]:\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n        ...    def get_test_data(self, indices=None):\n        ...        if indices is None:\n        ...            return self.x_test, self.y_test\n        ...        fraction = len(list(indices)) / len(self)\n        ...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n        ...        mapped_indices = np.unique(mapped_indices.astype(int))\n        ...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n        ...\n        &gt;&gt;&gt; X = np.random.rand(100, 10)\n        &gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n        &gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n        &gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n        &gt;&gt;&gt; _ = dataset.get_training_data(indices)\n        &gt;&gt;&gt; _ = dataset.get_test_data(indices)\n        ```\n\n    Args:\n        indices: Optional indices into the test data. This argument is\n            unused left for compatibility with\n            [get_training_data()][pydvl.utils.dataset.Dataset.get_training_data].\n\n    Returns:\n        The entire test data.\n    \"\"\"\n    return self.x_test, self.y_test\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.get_training_data","title":"get_training_data","text":"<pre><code>get_training_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Returns the data and labels of all samples in the given groups.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>group indices whose elements to return. If <code>None</code>, all data from all groups are returned.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>Tuple of training data x and labels y.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_training_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Returns the data and labels of all samples in the given groups.\n\n    Args:\n        indices: group indices whose elements to return. If `None`,\n            all data from all groups are returned.\n\n    Returns:\n        Tuple of training data x and labels y.\n    \"\"\"\n    if indices is None:\n        indices = self.indices\n    data_indices = [\n        idx for group_id in indices for idx in self.group_items[group_id][1]\n    ]\n    return super().get_training_data(data_indices)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_sklearn","title":"from_sklearn  <code>classmethod</code>","text":"<pre><code>from_sklearn(\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    data_groups: Optional[Sequence] = None,\n    **kwargs\n) -&gt; GroupedDataset\n</code></pre> <p>Constructs a GroupedDataset object from a sklearn.utils.Bunch as returned by the <code>load_*</code> functions in scikit-learn toy datasets and groups it.</p> Example <pre><code>&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; from pydvl.utils import GroupedDataset\n&gt;&gt;&gt; iris = load_iris()\n&gt;&gt;&gt; data_groups = iris.data[:, 0] // 0.5\n&gt;&gt;&gt; dataset = GroupedDataset.from_sklearn(iris, data_groups=data_groups)\n</code></pre> PARAMETER  DESCRIPTION <code>data</code> <p>scikit-learn Bunch object. The following attributes are supported:</p> <ul> <li><code>data</code>: covariates.</li> <li><code>target</code>: target variables (labels).</li> <li><code>feature_names</code> (optional): the feature names.</li> <li><code>target_names</code> (optional): the target names.</li> <li><code>DESCR</code> (optional): a description.</li> </ul> <p> TYPE: <code>Bunch</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the target variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>data_groups</code> <p>an array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Optional[Sequence]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>GroupedDataset</code> <p>Dataset with the selected sklearn data</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_sklearn(\n    cls,\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    data_groups: Optional[Sequence] = None,\n    **kwargs,\n) -&gt; \"GroupedDataset\":\n    \"\"\"Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from a\n    [sklearn.utils.Bunch][sklearn.utils.Bunch] as returned by the `load_*` functions in\n    [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html) and groups\n    it.\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from sklearn.datasets import load_iris\n        &gt;&gt;&gt; from pydvl.utils import GroupedDataset\n        &gt;&gt;&gt; iris = load_iris()\n        &gt;&gt;&gt; data_groups = iris.data[:, 0] // 0.5\n        &gt;&gt;&gt; dataset = GroupedDataset.from_sklearn(iris, data_groups=data_groups)\n        ```\n\n    Args:\n        data: scikit-learn Bunch object. The following attributes are supported:\n\n            - `data`: covariates.\n            - `target`: target variables (labels).\n            - `feature_names` (**optional**): the feature names.\n            - `target_names` (**optional**): the target names.\n            - `DESCR` (**optional**): a description.\n        train_size: size of the training dataset. Used in `train_test_split`.\n        random_state: seed for train / test split.\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the target variable as labels. Read more in\n            [sklearn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        data_groups: an array holding the group index or name for each\n            data point. The length of this array must be equal to the number of\n            data points in the dataset.\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n\n    Returns:\n        Dataset with the selected sklearn data\n    \"\"\"\n    if data_groups is None:\n        raise ValueError(\n            \"data_groups must be provided when constructing a GroupedDataset\"\n        )\n\n    x_train, x_test, y_train, y_test, data_groups_train, _ = train_test_split(\n        data.data,\n        data.target,\n        data_groups,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=data.target if stratify_by_target else None,\n    )\n\n    dataset = Dataset(\n        x_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test, **kwargs\n    )\n    return cls.from_dataset(dataset, data_groups_train)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_arrays","title":"from_arrays  <code>classmethod</code>","text":"<pre><code>from_arrays(\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    data_groups: Optional[Sequence] = None,\n    **kwargs\n) -&gt; Dataset\n</code></pre> <p>Constructs a GroupedDataset object from X and y numpy arrays as returned by the <code>make_*</code> functions in scikit-learn generated datasets.</p> Example <pre><code>&gt;&gt;&gt; from sklearn.datasets import make_classification\n&gt;&gt;&gt; from pydvl.utils import GroupedDataset\n&gt;&gt;&gt; X, y = make_classification(\n...     n_samples=100,\n...     n_features=4,\n...     n_informative=2,\n...     n_redundant=0,\n...     random_state=0,\n...     shuffle=False\n... )\n&gt;&gt;&gt; data_groups = X[:, 0] // 0.5\n&gt;&gt;&gt; dataset = GroupedDataset.from_arrays(X, y, data_groups=data_groups)\n</code></pre> PARAMETER  DESCRIPTION <code>X</code> <p>array of shape (n_samples, n_features)</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>array of shape (n_samples,)</p> <p> TYPE: <code>NDArray</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the y variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>data_groups</code> <p>an array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Optional[Sequence]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments that will be passed to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Dataset with the passed X and y arrays split across training and test sets.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_arrays(\n    cls,\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    data_groups: Optional[Sequence] = None,\n    **kwargs,\n) -&gt; \"Dataset\":\n    \"\"\"Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from X and y numpy arrays\n    as returned by the `make_*` functions in\n    [scikit-learn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from sklearn.datasets import make_classification\n        &gt;&gt;&gt; from pydvl.utils import GroupedDataset\n        &gt;&gt;&gt; X, y = make_classification(\n        ...     n_samples=100,\n        ...     n_features=4,\n        ...     n_informative=2,\n        ...     n_redundant=0,\n        ...     random_state=0,\n        ...     shuffle=False\n        ... )\n        &gt;&gt;&gt; data_groups = X[:, 0] // 0.5\n        &gt;&gt;&gt; dataset = GroupedDataset.from_arrays(X, y, data_groups=data_groups)\n        ```\n\n    Args:\n        X: array of shape (n_samples, n_features)\n        y: array of shape (n_samples,)\n        train_size: size of the training dataset. Used in `train_test_split`.\n        random_state: seed for train / test split.\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the y variable as labels. Read more in\n            [sklearn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        data_groups: an array holding the group index or name for each data\n            point. The length of this array must be equal to the number of\n            data points in the dataset.\n        kwargs: Additional keyword arguments that will be passed to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n\n    Returns:\n        Dataset with the passed X and y arrays split across training and\n            test sets.\n\n    !!! tip \"New in version 0.4.0\"\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\n    if data_groups is None:\n        raise ValueError(\n            \"data_groups must be provided when constructing a GroupedDataset\"\n        )\n    x_train, x_test, y_train, y_test, data_groups_train, _ = train_test_split(\n        X,\n        y,\n        data_groups,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=y if stratify_by_target else None,\n    )\n    dataset = Dataset(\n        x_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test, **kwargs\n    )\n    return cls.from_dataset(dataset, data_groups_train)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_dataset","title":"from_dataset  <code>classmethod</code>","text":"<pre><code>from_dataset(dataset: Dataset, data_groups: Sequence[Any]) -&gt; GroupedDataset\n</code></pre> <p>Creates a GroupedDataset object from the data a Dataset object and a mapping of data groups.</p> Example <pre><code>&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pydvl.utils import Dataset, GroupedDataset\n&gt;&gt;&gt; dataset = Dataset.from_arrays(\n...     X=np.asarray([[1, 2], [3, 4], [5, 6], [7, 8]]),\n...     y=np.asarray([0, 1, 0, 1]),\n... )\n&gt;&gt;&gt; dataset = GroupedDataset.from_dataset(dataset, data_groups=[0, 0, 1, 1])\n</code></pre> PARAMETER  DESCRIPTION <code>dataset</code> <p>The original data.</p> <p> TYPE: <code>Dataset</code> </p> <code>data_groups</code> <p>An array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Sequence[Any]</code> </p> RETURNS DESCRIPTION <code>GroupedDataset</code> <p>A GroupedDataset with the initial Dataset grouped by data_groups.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_dataset(\n    cls, dataset: Dataset, data_groups: Sequence[Any]\n) -&gt; \"GroupedDataset\":\n    \"\"\"Creates a [GroupedDataset][pydvl.utils.GroupedDataset] object from the data a\n    [Dataset][pydvl.utils.Dataset] object and a mapping of data groups.\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; from pydvl.utils import Dataset, GroupedDataset\n        &gt;&gt;&gt; dataset = Dataset.from_arrays(\n        ...     X=np.asarray([[1, 2], [3, 4], [5, 6], [7, 8]]),\n        ...     y=np.asarray([0, 1, 0, 1]),\n        ... )\n        &gt;&gt;&gt; dataset = GroupedDataset.from_dataset(dataset, data_groups=[0, 0, 1, 1])\n        ```\n\n    Args:\n        dataset: The original data.\n        data_groups: An array holding the group index or name for each data\n            point. The length of this array must be equal to the number of\n            data points in the dataset.\n\n    Returns:\n        A [GroupedDataset][pydvl.utils.GroupedDataset] with the initial\n            [Dataset][pydvl.utils.Dataset] grouped by data_groups.\n    \"\"\"\n    return cls(\n        x_train=dataset.x_train,\n        y_train=dataset.y_train,\n        x_test=dataset.x_test,\n        y_test=dataset.y_test,\n        data_groups=data_groups,\n        feature_names=dataset.feature_names,\n        target_names=dataset.target_names,\n        description=dataset.description,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/functional/","title":"Functional","text":""},{"location":"api/pydvl/utils/functional/#pydvl.utils.functional","title":"pydvl.utils.functional","text":"<p>Supporting utilities for manipulating arguments of functions.</p>"},{"location":"api/pydvl/utils/functional/#pydvl.utils.functional.free_arguments","title":"free_arguments","text":"<pre><code>free_arguments(fun: Union[Callable, partial]) -&gt; Set[str]\n</code></pre> <p>Computes the set of free arguments for a function or functools.partial object.</p> <p>All arguments of a function are considered free unless they are set by a partial. For example, if <code>f = partial(g, a=1)</code>, then <code>a</code> is not a free argument of <code>f</code>.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>A callable or a [partial object][].</p> <p> TYPE: <code>Union[Callable, partial]</code> </p> RETURNS DESCRIPTION <code>Set[str]</code> <p>The set of free arguments of <code>fun</code>.</p> <p>New in version 0.7.0</p> Source code in <code>src/pydvl/utils/functional.py</code> <pre><code>def free_arguments(fun: Union[Callable, partial]) -&gt; Set[str]:\n    \"\"\"Computes the set of free arguments for a function or\n    [functools.partial][] object.\n\n    All arguments of a function are considered free unless they are set by a\n    partial. For example, if `f = partial(g, a=1)`, then `a` is not a free\n    argument of `f`.\n\n    Args:\n        fun: A callable or a [partial object][].\n\n    Returns:\n        The set of free arguments of `fun`.\n\n    !!! tip \"New in version 0.7.0\"\n    \"\"\"\n    args_set_by_partial: Set[str] = set()\n\n    def _rec_unroll_partial_function_args(g: Union[Callable, partial]) -&gt; Callable:\n        \"\"\"Stores arguments and recursively call itself if `g` is a\n        [functools.partial][] object. In the end, returns the initially wrapped\n        function.\n\n        This handles the construct `partial(_accept_additional_argument, *args,\n        **kwargs)` that is used by `maybe_add_argument`.\n\n        Args:\n            g: A partial or a function to unroll.\n\n        Returns:\n            Initial wrapped function.\n        \"\"\"\n        nonlocal args_set_by_partial\n\n        if isinstance(g, partial) and g.func == _accept_additional_argument:\n            arg = g.keywords[\"arg\"]\n            if arg in args_set_by_partial:\n                args_set_by_partial.remove(arg)\n            return _rec_unroll_partial_function_args(g.keywords[\"fun\"])\n        elif isinstance(g, partial):\n            args_set_by_partial.update(g.keywords.keys())\n            args_set_by_partial.update(g.args)\n            return _rec_unroll_partial_function_args(g.func)\n        else:\n            return g\n\n    wrapped_fn = _rec_unroll_partial_function_args(fun)\n    sig = inspect.signature(wrapped_fn)\n    return args_set_by_partial | set(sig.parameters.keys())\n</code></pre>"},{"location":"api/pydvl/utils/functional/#pydvl.utils.functional.maybe_add_argument","title":"maybe_add_argument","text":"<pre><code>maybe_add_argument(fun: Callable, new_arg: str) -&gt; Callable\n</code></pre> <p>Wraps a function to accept the given keyword parameter if it doesn't already.</p> <p>If <code>fun</code> already takes a keyword parameter of name <code>new_arg</code>, then it is returned as is. Otherwise, a wrapper is returned which merely ignores the argument.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>The function to wrap</p> <p> TYPE: <code>Callable</code> </p> <code>new_arg</code> <p>The name of the argument that the new function will accept (and ignore).</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>A new function accepting one more keyword argument.</p> <p>Changed in version 0.7.0</p> <p>Ability to work with partials.</p> Source code in <code>src/pydvl/utils/functional.py</code> <pre><code>def maybe_add_argument(fun: Callable, new_arg: str) -&gt; Callable:\n    \"\"\"Wraps a function to accept the given keyword parameter if it doesn't\n    already.\n\n    If `fun` already takes a keyword parameter of name `new_arg`, then it is\n    returned as is. Otherwise, a wrapper is returned which merely ignores the\n    argument.\n\n    Args:\n        fun: The function to wrap\n        new_arg: The name of the argument that the new function will accept\n            (and ignore).\n\n    Returns:\n        A new function accepting one more keyword argument.\n\n    !!! tip \"Changed in version 0.7.0\"\n        Ability to work with partials.\n    \"\"\"\n    if new_arg in free_arguments(fun):\n        return fun\n\n    return partial(_accept_additional_argument, fun=fun, arg=new_arg)\n</code></pre>"},{"location":"api/pydvl/utils/numeric/","title":"Numeric","text":""},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric","title":"pydvl.utils.numeric","text":"<p>This module contains routines for numerical computations used across the library.</p>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.powerset","title":"powerset","text":"<pre><code>powerset(s: NDArray[T]) -&gt; Iterator[Collection[T]]\n</code></pre> <p>Returns an iterator for the power set of the argument.</p> <p>Subsets are generated in sequence by growing size. See  random_powerset() for random  sampling.</p> Example <pre><code>&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pydvl.utils.numeric import powerset\n&gt;&gt;&gt; list(powerset(np.array((1,2))))\n[(), (1,), (2,), (1, 2)]\n</code></pre> PARAMETER  DESCRIPTION <code>s</code> <p>The set to use</p> <p> TYPE: <code>NDArray[T]</code> </p> RETURNS DESCRIPTION <code>Iterator[Collection[T]]</code> <p>An iterator over all subsets of the set of indices <code>s</code>.</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def powerset(s: NDArray[T]) -&gt; Iterator[Collection[T]]:\n    \"\"\"Returns an iterator for the power set of the argument.\n\n     Subsets are generated in sequence by growing size. See\n     [random_powerset()][pydvl.utils.numeric.random_powerset] for random\n     sampling.\n\n    ??? Example\n        ``` pycon\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; from pydvl.utils.numeric import powerset\n        &gt;&gt;&gt; list(powerset(np.array((1,2))))\n        [(), (1,), (2,), (1, 2)]\n        ```\n\n    Args:\n         s: The set to use\n\n    Returns:\n        An iterator over all subsets of the set of indices `s`.\n    \"\"\"\n    return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1))\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.num_samples_permutation_hoeffding","title":"num_samples_permutation_hoeffding","text":"<pre><code>num_samples_permutation_hoeffding(\n    eps: float, delta: float, u_range: float\n) -&gt; int\n</code></pre> <p>Lower bound on the number of samples required for MonteCarlo Shapley to obtain an (\u03b5,\u03b4)-approximation.</p> <p>That is: with probability 1-\u03b4, the estimated value for one data point will be \u03b5-close to the true quantity, if at least this many permutations are sampled.</p> PARAMETER  DESCRIPTION <code>eps</code> <p>\u03b5 &gt; 0</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>0 &lt; \u03b4 &lt;= 1</p> <p> TYPE: <code>float</code> </p> <code>u_range</code> <p>Range of the Utility function</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>int</code> <p>Number of permutations required to guarantee \u03b5-correct Shapley values with probability 1-\u03b4</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def num_samples_permutation_hoeffding(eps: float, delta: float, u_range: float) -&gt; int:\n    \"\"\"Lower bound on the number of samples required for MonteCarlo Shapley to\n    obtain an (\u03b5,\u03b4)-approximation.\n\n    That is: with probability 1-\u03b4, the estimated value for one data point will\n    be \u03b5-close to the true quantity, if at least this many permutations are\n    sampled.\n\n    Args:\n        eps: \u03b5 &gt; 0\n        delta: 0 &lt; \u03b4 &lt;= 1\n        u_range: Range of the [Utility][pydvl.utils.utility.Utility] function\n\n    Returns:\n        Number of _permutations_ required to guarantee \u03b5-correct Shapley\n            values with probability 1-\u03b4\n    \"\"\"\n    return int(np.ceil(np.log(2 / delta) * 2 * u_range**2 / eps**2))\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_subset","title":"random_subset","text":"<pre><code>random_subset(\n    s: NDArray[T], q: float = 0.5, seed: Optional[Seed] = None\n) -&gt; NDArray[T]\n</code></pre> <p>Returns one subset at random from <code>s</code>.</p> PARAMETER  DESCRIPTION <code>s</code> <p>set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>q</code> <p>Sampling probability for elements. The default 0.5 yields a uniform distribution over the power set of s.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.5</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>The subset</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_subset(\n    s: NDArray[T], q: float = 0.5, seed: Optional[Seed] = None\n) -&gt; NDArray[T]:\n    \"\"\"Returns one subset at random from ``s``.\n\n    Args:\n        s: set to sample from\n        q: Sampling probability for elements. The default 0.5 yields a\n            uniform distribution over the power set of s.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n\n    Returns:\n        The subset\n    \"\"\"\n    rng = np.random.default_rng(seed)\n    selection = rng.uniform(size=len(s)) &gt; q\n    return s[selection]\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_powerset","title":"random_powerset","text":"<pre><code>random_powerset(\n    s: NDArray[T],\n    n_samples: Optional[int] = None,\n    q: float = 0.5,\n    seed: Optional[Seed] = None,\n) -&gt; Generator[NDArray[T], None, None]\n</code></pre> <p>Samples subsets from the power set of the argument, without pre-generating all subsets and in no order.</p> <p>See powerset if you wish to deterministically generate all subsets.</p> <p>To generate subsets, <code>len(s)</code> Bernoulli draws with probability <code>q</code> are drawn. The default value of <code>q = 0.5</code> provides a uniform distribution over the power set of <code>s</code>. Other choices can be used e.g. to implement owen_sampling_shapley.</p> PARAMETER  DESCRIPTION <code>s</code> <p>set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>n_samples</code> <p>if set, stop the generator after this many steps. Defaults to <code>np.iinfo(np.int32).max</code></p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>q</code> <p>Sampling probability for elements. The default 0.5 yields a uniform distribution over the power set of s.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.5</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Generator[NDArray[T], None, None]</code> <p>Samples from the power set of <code>s</code>.</p> RAISES DESCRIPTION <code>ValueError</code> <p>if the element sampling probability is not in [0,1]</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_powerset(\n    s: NDArray[T],\n    n_samples: Optional[int] = None,\n    q: float = 0.5,\n    seed: Optional[Seed] = None,\n) -&gt; Generator[NDArray[T], None, None]:\n    \"\"\"Samples subsets from the power set of the argument, without\n    pre-generating all subsets and in no order.\n\n    See [powerset][pydvl.utils.numeric.powerset] if you wish to deterministically generate all subsets.\n\n    To generate subsets, `len(s)` Bernoulli draws with probability `q` are\n    drawn. The default value of `q = 0.5` provides a uniform distribution over\n    the power set of `s`. Other choices can be used e.g. to implement\n    [owen_sampling_shapley][pydvl.value.shapley.owen.owen_sampling_shapley].\n\n    Args:\n        s: set to sample from\n        n_samples: if set, stop the generator after this many steps.\n            Defaults to `np.iinfo(np.int32).max`\n        q: Sampling probability for elements. The default 0.5 yields a\n            uniform distribution over the power set of s.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Samples from the power set of `s`.\n\n    Raises:\n        ValueError: if the element sampling probability is not in [0,1]\n\n    \"\"\"\n    if q &lt; 0 or q &gt; 1:\n        raise ValueError(\"Element sampling probability must be in [0,1]\")\n\n    rng = np.random.default_rng(seed)\n    total = 1\n    if n_samples is None:\n        n_samples = np.iinfo(np.int32).max\n    while total &lt;= n_samples:\n        yield random_subset(s, q, seed=rng)\n        total += 1\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_powerset_label_min","title":"random_powerset_label_min","text":"<pre><code>random_powerset_label_min(\n    s: NDArray[T],\n    labels: NDArray[int_],\n    min_elements_per_label: int = 1,\n    seed: Optional[Seed] = None,\n) -&gt; Generator[NDArray[T], None, None]\n</code></pre> <p>Draws random subsets from <code>s</code>, while ensuring that at least <code>min_elements_per_label</code> elements per label are included in the draw. It can be used for classification problems to ensure that a set contains information for all labels (or not if <code>min_elements_per_label=0</code>).</p> PARAMETER  DESCRIPTION <code>s</code> <p>Set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>labels</code> <p>Labels for the samples</p> <p> TYPE: <code>NDArray[int_]</code> </p> <code>min_elements_per_label</code> <p>Minimum number of elements for each label.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Generator[NDArray[T], None, None]</code> <p>Generated draw from the powerset of s with <code>min_elements_per_label</code> for each</p> <code>Generator[NDArray[T], None, None]</code> <p>label.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>s</code> and <code>labels</code> are of different length or <code>min_elements_per_label</code> is smaller than 0.</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_powerset_label_min(\n    s: NDArray[T],\n    labels: NDArray[np.int_],\n    min_elements_per_label: int = 1,\n    seed: Optional[Seed] = None,\n) -&gt; Generator[NDArray[T], None, None]:\n    \"\"\"Draws random subsets from `s`, while ensuring that at least\n    `min_elements_per_label` elements per label are included in the draw. It can be used\n    for classification problems to ensure that a set contains information for all labels\n    (or not if `min_elements_per_label=0`).\n\n    Args:\n        s: Set to sample from\n        labels: Labels for the samples\n        min_elements_per_label: Minimum number of elements for each label.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Generated draw from the powerset of s with `min_elements_per_label` for each\n        label.\n\n    Raises:\n        ValueError: If `s` and `labels` are of different length or\n            `min_elements_per_label` is smaller than 0.\n    \"\"\"\n    if len(labels) != len(s):\n        raise ValueError(\"Set and labels have to be of same size.\")\n\n    if min_elements_per_label &lt; 0:\n        raise ValueError(\n            f\"Parameter min_elements={min_elements_per_label} needs to be bigger or \"\n            f\"equal to 0.\"\n        )\n\n    rng = np.random.default_rng(seed)\n    unique_labels = np.unique(labels)\n\n    while True:\n        subsets: List[NDArray[T]] = []\n        for label in unique_labels:\n            label_indices = np.asarray(np.where(labels == label)[0])\n            subset_size = int(\n                rng.integers(\n                    min(min_elements_per_label, len(label_indices)),\n                    len(label_indices) + 1,\n                )\n            )\n            if subset_size &gt; 0:\n                subsets.append(\n                    random_subset_of_size(s[label_indices], subset_size, seed=rng)\n                )\n\n        if len(subsets) &gt; 0:\n            subset = np.concatenate(tuple(subsets))\n            rng.shuffle(subset)\n            yield subset\n        else:\n            yield np.array([], dtype=s.dtype)\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_subset_of_size","title":"random_subset_of_size","text":"<pre><code>random_subset_of_size(\n    s: NDArray[T], size: int, seed: Optional[Seed] = None\n) -&gt; NDArray[T]\n</code></pre> <p>Samples a random subset of given size uniformly from the powerset of <code>s</code>.</p> PARAMETER  DESCRIPTION <code>s</code> <p>Set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>size</code> <p>Size of the subset to generate</p> <p> TYPE: <code>int</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>The subset</p> <p>Raises     ValueError: If size &gt; len(s)</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_subset_of_size(\n    s: NDArray[T], size: int, seed: Optional[Seed] = None\n) -&gt; NDArray[T]:\n    \"\"\"Samples a random subset of given size uniformly from the powerset\n    of `s`.\n\n    Args:\n        s: Set to sample from\n        size: Size of the subset to generate\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        The subset\n\n    Raises\n        ValueError: If size &gt; len(s)\n    \"\"\"\n    if size &gt; len(s):\n        raise ValueError(\"Cannot sample subset larger than set\")\n    rng = np.random.default_rng(seed)\n    return rng.choice(s, size=size, replace=False)\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_matrix_with_condition_number","title":"random_matrix_with_condition_number","text":"<pre><code>random_matrix_with_condition_number(\n    n: int, condition_number: float, seed: Optional[Seed] = None\n) -&gt; NDArray\n</code></pre> <p>Constructs a square matrix with a given condition number.</p> <p>Taken from: https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py</p> <p>Also see: https://math.stackexchange.com/questions/1351616/condition-number-of-ata.</p> PARAMETER  DESCRIPTION <code>n</code> <p>size of the matrix</p> <p> TYPE: <code>int</code> </p> <code>condition_number</code> <p>duh</p> <p> TYPE: <code>float</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray</code> <p>An (n,n) matrix with the requested condition number.</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_matrix_with_condition_number(\n    n: int, condition_number: float, seed: Optional[Seed] = None\n) -&gt; NDArray:\n    \"\"\"Constructs a square matrix with a given condition number.\n\n    Taken from:\n    [https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py](\n    https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py)\n\n    Also see:\n    [https://math.stackexchange.com/questions/1351616/condition-number-of-ata](\n    https://math.stackexchange.com/questions/1351616/condition-number-of-ata).\n\n    Args:\n        n: size of the matrix\n        condition_number: duh\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        An (n,n) matrix with the requested condition number.\n    \"\"\"\n    if n &lt; 2:\n        raise ValueError(\"Matrix size must be at least 2\")\n\n    if condition_number &lt;= 1:\n        raise ValueError(\"Condition number must be greater than 1\")\n\n    rng = np.random.default_rng(seed)\n    log_condition_number = np.log(condition_number)\n    exp_vec = np.arange(\n        -log_condition_number / 4.0,\n        log_condition_number * (n + 1) / (4 * (n - 1)),\n        log_condition_number / (2.0 * (n - 1)),\n    )\n    exp_vec = exp_vec[:n]\n    s: np.ndarray = np.exp(exp_vec)\n    S = np.diag(s)\n    U, _ = np.linalg.qr((rng.uniform(size=(n, n)) - 5.0) * 200)\n    V, _ = np.linalg.qr((rng.uniform(size=(n, n)) - 5.0) * 200)\n    P: np.ndarray = U.dot(S).dot(V.T)\n    P = P.dot(P.T)\n    return P\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.running_moments","title":"running_moments","text":"<pre><code>running_moments(\n    previous_avg: float | NDArray[float_],\n    previous_variance: float | NDArray[float_],\n    count: int,\n    new_value: float | NDArray[float_],\n) -&gt; Tuple[float | NDArray[float_], float | NDArray[float_]]\n</code></pre> <p>Uses Welford's algorithm to calculate the running average and variance of  a set of numbers.</p> <p>See Welford's algorithm in wikipedia</p> <p>Warning</p> <p>This is not really using Welford's correction for numerical stability for the variance. (FIXME)</p> <p>Todo</p> <p>This could be generalised to arbitrary moments. See this paper</p> PARAMETER  DESCRIPTION <code>previous_avg</code> <p>average value at previous step</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> <code>previous_variance</code> <p>variance at previous step</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> <code>count</code> <p>number of points seen so far</p> <p> TYPE: <code>int</code> </p> <code>new_value</code> <p>new value in the series of numbers</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> RETURNS DESCRIPTION <code>Tuple[float | NDArray[float_], float | NDArray[float_]]</code> <p>new_average, new_variance, calculated with the new count</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def running_moments(\n    previous_avg: float | NDArray[np.float_],\n    previous_variance: float | NDArray[np.float_],\n    count: int,\n    new_value: float | NDArray[np.float_],\n) -&gt; Tuple[float | NDArray[np.float_], float | NDArray[np.float_]]:\n    \"\"\"Uses Welford's algorithm to calculate the running average and variance of\n     a set of numbers.\n\n    See [Welford's algorithm in wikipedia](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm)\n\n    !!! Warning\n        This is not really using Welford's correction for numerical stability\n        for the variance. (FIXME)\n\n    !!! Todo\n        This could be generalised to arbitrary moments. See [this paper](https://www.osti.gov/biblio/1028931)\n\n    Args:\n        previous_avg: average value at previous step\n        previous_variance: variance at previous step\n        count: number of points seen so far\n        new_value: new value in the series of numbers\n\n    Returns:\n        new_average, new_variance, calculated with the new count\n    \"\"\"\n    # broadcasted operations seem not to be supported by mypy, so we ignore the type\n    new_average = (new_value + count * previous_avg) / (count + 1)  # type: ignore\n    new_variance = previous_variance + (\n        (new_value - previous_avg) * (new_value - new_average) - previous_variance\n    ) / (count + 1)\n    return new_average, new_variance\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.top_k_value_accuracy","title":"top_k_value_accuracy","text":"<pre><code>top_k_value_accuracy(\n    y_true: NDArray[float_], y_pred: NDArray[float_], k: int = 3\n) -&gt; float\n</code></pre> <p>Computes the top-k accuracy for the estimated values by comparing indices of the highest k values.</p> PARAMETER  DESCRIPTION <code>y_true</code> <p>Exact/true value</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>y_pred</code> <p>Predicted/estimated value</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>k</code> <p>Number of the highest values taken into account</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p> RETURNS DESCRIPTION <code>float</code> <p>Accuracy</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def top_k_value_accuracy(\n    y_true: NDArray[np.float_], y_pred: NDArray[np.float_], k: int = 3\n) -&gt; float:\n    \"\"\"Computes the top-k accuracy for the estimated values by comparing indices\n    of the highest k values.\n\n    Args:\n        y_true: Exact/true value\n        y_pred: Predicted/estimated value\n        k: Number of the highest values taken into account\n\n    Returns:\n        Accuracy\n    \"\"\"\n    top_k_exact_values = np.argsort(y_true)[-k:]\n    top_k_pred_values = np.argsort(y_pred)[-k:]\n    top_k_accuracy = len(np.intersect1d(top_k_exact_values, top_k_pred_values)) / k\n    return top_k_accuracy\n</code></pre>"},{"location":"api/pydvl/utils/progress/","title":"Progress","text":""},{"location":"api/pydvl/utils/progress/#pydvl.utils.progress","title":"pydvl.utils.progress","text":""},{"location":"api/pydvl/utils/progress/#pydvl.utils.progress.repeat_indices","title":"repeat_indices","text":"<pre><code>repeat_indices(\n    indices: Collection[int],\n    result: ValuationResult,\n    done: StoppingCriterion,\n    **kwargs\n) -&gt; Iterator[int]\n</code></pre> <p>Helper function to cycle indefinitely over a collection of indices until the stopping criterion is satisfied while displaying progress.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>Collection of indices that will be cycled until done.</p> <p> TYPE: <code>Collection[int]</code> </p> <code>result</code> <p>Object containing the current results.</p> <p> TYPE: <code>ValuationResult</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>kwargs</code> <p>Keyword arguments passed to tqdm.</p> <p> DEFAULT: <code>{}</code> </p> Source code in <code>src/pydvl/utils/progress.py</code> <pre><code>def repeat_indices(\n    indices: Collection[int],\n    result: \"ValuationResult\",\n    done: \"StoppingCriterion\",\n    **kwargs,\n) -&gt; Iterator[int]:\n    \"\"\"Helper function to cycle indefinitely over a collection of indices\n    until the stopping criterion is satisfied while displaying progress.\n\n    Args:\n        indices: Collection of indices that will be cycled until done.\n        result: Object containing the current results.\n        done: Stopping criterion.\n        kwargs: Keyword arguments passed to tqdm.\n    \"\"\"\n    with tqdm(total=100, unit=\"%\", **kwargs) as pbar:\n        it = takewhile(lambda _: not done(result), cycle(indices))\n        for i in it:\n            yield i\n            pbar.update(100 * done.completion() - pbar.n)\n            pbar.refresh()\n</code></pre>"},{"location":"api/pydvl/utils/progress/#pydvl.utils.progress.log_duration","title":"log_duration","text":"<pre><code>log_duration(func)\n</code></pre> <p>Decorator to log execution time of a function</p> Source code in <code>src/pydvl/utils/progress.py</code> <pre><code>def log_duration(func):\n    \"\"\"\n    Decorator to log execution time of a function\n    \"\"\"\n\n    @wraps(func)\n    def wrapper_log_duration(*args, **kwargs):\n        func_name = func.__qualname__\n        logger.info(f\"Function '{func_name}' is starting.\")\n        start_time = time()\n        result = func(*args, **kwargs)\n        duration = time() - start_time\n        logger.info(f\"Function '{func_name}' completed. Duration: {duration:.2f} sec\")\n        return result\n\n    return wrapper_log_duration\n</code></pre>"},{"location":"api/pydvl/utils/score/","title":"Score","text":""},{"location":"api/pydvl/utils/score/#pydvl.utils.score","title":"pydvl.utils.score","text":"<p>This module provides a Scorer class that wraps scoring functions with additional information.</p> <p>Scorers are the fundamental building block of many data valuation methods. They are typically used by the Utility class to evaluate the quality of a model when trained on subsets of the training data.</p> <p>Scorers can be constructed in the same way as in scikit-learn: either from  known strings or from a callable. Greater values must be better. If they are not, a negated version can be used, see scikit-learn's make_scorer().</p> <p>Scorer provides additional information about the scoring function, like its range and default values, which can be used by some data valuation methods (like group_testing_shapley()) to estimate the number of samples required for a certain quality of approximation.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.squashed_r2","title":"squashed_r2  <code>module-attribute</code>","text":"<pre><code>squashed_r2 = compose_score(Scorer('r2'), _sigmoid, (0, 1), 'squashed r2')\n</code></pre> <p>A scorer that squashes the R\u00b2 score into the range [0, 1] using a sigmoid.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.squashed_variance","title":"squashed_variance  <code>module-attribute</code>","text":"<pre><code>squashed_variance = compose_score(\n    Scorer(\"explained_variance\"),\n    _sigmoid,\n    (0, 1),\n    \"squashed explained variance\",\n)\n</code></pre> <p>A scorer that squashes the explained variance score into the range [0, 1] using a sigmoid.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.ScorerCallable","title":"ScorerCallable","text":"<p>             Bases: <code>Protocol</code></p> <p>Signature for a scorer</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.Scorer","title":"Scorer","text":"<pre><code>Scorer(\n    scoring: Union[str, ScorerCallable],\n    default: float = np.nan,\n    range: Tuple = (-np.inf, np.inf),\n    name: Optional[str] = None,\n)\n</code></pre> <p>A scoring callable that takes a model, data, and labels and returns a scalar.</p> PARAMETER  DESCRIPTION <code>scoring</code> <p>Either a string or callable that can be passed to get_scorer.</p> <p> TYPE: <code>Union[str, ScorerCallable]</code> </p> <code>default</code> <p>score to be used when a model cannot be fit, e.g. when too little data is passed, or errors arise.</p> <p> TYPE: <code>float</code> DEFAULT: <code>nan</code> </p> <code>range</code> <p>numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation. If not provided, it can be read from the <code>scoring</code> object if it provides it, for instance if it was constructed with compose_score().</p> <p> TYPE: <code>Tuple</code> DEFAULT: <code>(-inf, inf)</code> </p> <code>name</code> <p>The name of the scorer. If not provided, the name of the function passed will be used.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <p>New in version 0.5.0</p> Source code in <code>src/pydvl/utils/score.py</code> <pre><code>def __init__(\n    self,\n    scoring: Union[str, ScorerCallable],\n    default: float = np.nan,\n    range: Tuple = (-np.inf, np.inf),\n    name: Optional[str] = None,\n):\n    if name is None and isinstance(scoring, str):\n        name = scoring\n    self._scorer = get_scorer(scoring)\n    self.default = default\n    # TODO: auto-fill from known scorers ?\n    self.range = np.array(range)\n    self._name = getattr(self._scorer, \"__name__\", name or \"scorer\")\n</code></pre>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.compose_score","title":"compose_score","text":"<pre><code>compose_score(\n    scorer: Scorer,\n    transformation: Callable[[float], float],\n    range: Tuple[float, float],\n    name: str,\n) -&gt; Scorer\n</code></pre> <p>Composes a scoring function with an arbitrary scalar transformation.</p> <p>Useful to squash unbounded scores into ranges manageable by data valuation methods.</p> <p>Example:</p> <pre><code>sigmoid = lambda x: 1/(1+np.exp(-x))\ncompose_score(Scorer(\"r2\"), sigmoid, range=(0,1), name=\"squashed r2\")\n</code></pre> PARAMETER  DESCRIPTION <code>scorer</code> <p>The object to be composed.</p> <p> TYPE: <code>Scorer</code> </p> <code>transformation</code> <p>A scalar transformation</p> <p> TYPE: <code>Callable[[float], float]</code> </p> <code>range</code> <p>The range of the transformation. This will be used e.g. by Utility for the range of the composed.</p> <p> TYPE: <code>Tuple[float, float]</code> </p> <code>name</code> <p>A string representation for the composition, for <code>str()</code>.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Scorer</code> <p>The composite Scorer.</p> Source code in <code>src/pydvl/utils/score.py</code> <pre><code>def compose_score(\n    scorer: Scorer,\n    transformation: Callable[[float], float],\n    range: Tuple[float, float],\n    name: str,\n) -&gt; Scorer:\n    \"\"\"Composes a scoring function with an arbitrary scalar transformation.\n\n    Useful to squash unbounded scores into ranges manageable by data valuation\n    methods.\n\n    Example:\n\n    ```python\n    sigmoid = lambda x: 1/(1+np.exp(-x))\n    compose_score(Scorer(\"r2\"), sigmoid, range=(0,1), name=\"squashed r2\")\n    ```\n\n    Args:\n        scorer: The object to be composed.\n        transformation: A scalar transformation\n        range: The range of the transformation. This will be used e.g. by\n            [Utility][pydvl.utils.utility.Utility] for the range of the composed.\n        name: A string representation for the composition, for `str()`.\n\n    Returns:\n        The composite [Scorer][pydvl.utils.score.Scorer].\n    \"\"\"\n\n    class CompositeScorer(Scorer):\n        def __call__(self, model: SupervisedModel, X: NDArray, y: NDArray) -&gt; float:\n            score = self._scorer(model=model, X=X, y=y)\n            return transformation(score)\n\n    return CompositeScorer(scorer, range=range, name=name)\n</code></pre>"},{"location":"api/pydvl/utils/status/","title":"Status","text":""},{"location":"api/pydvl/utils/status/#pydvl.utils.status","title":"pydvl.utils.status","text":""},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status","title":"Status","text":"<p>             Bases: <code>Enum</code></p> <p>Status of a computation.</p> <p>Statuses can be combined using bitwise or (<code>|</code>) and bitwise and (<code>&amp;</code>) to get the status of a combined computation. For example, if we have two computations, one that has converged and one that has failed, then the combined status is <code>Status.Converged | Status.Failed == Status.Converged</code>, but <code>Status.Converged &amp; Status.Failed == Status.Failed</code>.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--or","title":"OR","text":"<p>The result of bitwise or-ing two valuation statuses with <code>|</code> is given by the following table:</p> P C F P P C P C C C C F P C F <p>where P = Pending, C = Converged, F = Failed.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--and","title":"AND","text":"<p>The result of bitwise and-ing two valuation statuses with <code>&amp;</code> is given by the following table:</p> P C F P P P F C P C F F F F F <p>where P = Pending, C = Converged, F = Failed.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--not","title":"NOT","text":"<p>The result of bitwise negation of a Status with <code>~</code> is <code>Failed</code> if the status is <code>Converged</code>, or <code>Converged</code> otherwise:</p> <pre><code>~P == C, ~C == F, ~F == C\n</code></pre>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--boolean-casting","title":"Boolean casting","text":"<p>A Status evaluates to <code>True</code> iff it's <code>Converged</code> or <code>Failed</code>:</p> <pre><code>bool(Status.Pending) == False\nbool(Status.Converged) == True\nbool(Status.Failed) == True\n</code></pre> <p>Warning</p> <p>These truth values are inconsistent with the usual boolean operations. In particular the XOR of two instances of <code>Status</code> is not the same as the XOR of their boolean values.</p>"},{"location":"api/pydvl/utils/types/","title":"Types","text":""},{"location":"api/pydvl/utils/types/#pydvl.utils.types","title":"pydvl.utils.types","text":"<p>This module contains types, protocols, decorators and generic function transformations. Some of it probably belongs elsewhere.</p>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel","title":"SupervisedModel","text":"<p>             Bases: <code>Protocol</code></p> <p>This is the minimal Protocol that valuation methods require from models in order to work.</p> <p>All that is needed are the standard sklearn methods <code>fit()</code>, <code>predict()</code> and <code>score()</code>.</p>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel.fit","title":"fit","text":"<pre><code>fit(x: NDArray, y: NDArray)\n</code></pre> <p>Fit the model to the data</p> PARAMETER  DESCRIPTION <code>x</code> <p>Independent variables</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>Dependent variable</p> <p> TYPE: <code>NDArray</code> </p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def fit(self, x: NDArray, y: NDArray):\n    \"\"\"Fit the model to the data\n\n    Args:\n        x: Independent variables\n        y: Dependent variable\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel.predict","title":"predict","text":"<pre><code>predict(x: NDArray) -&gt; NDArray\n</code></pre> <p>Compute predictions for the input</p> PARAMETER  DESCRIPTION <code>x</code> <p>Independent variables for which to compute predictions</p> <p> TYPE: <code>NDArray</code> </p> RETURNS DESCRIPTION <code>NDArray</code> <p>Predictions for the input</p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def predict(self, x: NDArray) -&gt; NDArray:\n    \"\"\"Compute predictions for the input\n\n    Args:\n        x: Independent variables for which to compute predictions\n\n    Returns:\n        Predictions for the input\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel.score","title":"score","text":"<pre><code>score(x: NDArray, y: NDArray) -&gt; float\n</code></pre> <p>Compute the score of the model given test data</p> PARAMETER  DESCRIPTION <code>x</code> <p>Independent variables</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>Dependent variable</p> <p> TYPE: <code>NDArray</code> </p> RETURNS DESCRIPTION <code>float</code> <p>The score of the model on <code>(x, y)</code></p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def score(self, x: NDArray, y: NDArray) -&gt; float:\n    \"\"\"Compute the score of the model given test data\n\n    Args:\n        x: Independent variables\n        y: Dependent variable\n\n    Returns:\n        The score of the model on `(x, y)`\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.ensure_seed_sequence","title":"ensure_seed_sequence","text":"<pre><code>ensure_seed_sequence(\n    seed: Optional[Union[Seed, SeedSequence]] = None\n) -&gt; SeedSequence\n</code></pre> <p>If the passed seed is a SeedSequence object then it is returned as is. If it is a Generator the internal protected seed sequence from the generator gets extracted. Otherwise, a new SeedSequence object is created from the passed (optional) seed.</p> PARAMETER  DESCRIPTION <code>seed</code> <p>Either an int, a Generator object a SeedSequence object or None.</p> <p> TYPE: <code>Optional[Union[Seed, SeedSequence]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>SeedSequence</code> <p>A SeedSequence object.</p> <p>New in version 0.7.0</p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def ensure_seed_sequence(\n    seed: Optional[Union[Seed, SeedSequence]] = None\n) -&gt; SeedSequence:\n    \"\"\"\n    If the passed seed is a SeedSequence object then it is returned as is. If it is\n    a Generator the internal protected seed sequence from the generator gets extracted.\n    Otherwise, a new SeedSequence object is created from the passed (optional) seed.\n\n    Args:\n        seed: Either an int, a Generator object a SeedSequence object or None.\n\n    Returns:\n        A SeedSequence object.\n\n    !!! tip \"New in version 0.7.0\"\n    \"\"\"\n    if isinstance(seed, SeedSequence):\n        return seed\n    elif isinstance(seed, Generator):\n        return cast(SeedSequence, seed.bit_generator.seed_seq)  # type: ignore\n    else:\n        return SeedSequence(seed)\n</code></pre>"},{"location":"api/pydvl/utils/utility/","title":"Utility","text":""},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility","title":"pydvl.utils.utility","text":"<p>This module contains classes to manage and learn utility functions for the computation of values. Please see the documentation on Computing Data Values for more information.</p> <p>Utility holds information about model, data and scoring function (the latter being what one usually understands under utility in the general definition of Shapley value). It is automatically cached across machines when the cache is configured and it is enabled upon construction.</p> <p>DataUtilityLearning adds support for learning the scoring function to avoid repeated re-training of the model to compute the score.</p> <p>This module also contains derived <code>Utility</code> classes for toy games that are used for testing and for demonstration purposes.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility--references","title":"References","text":"<ol> <li> <p>Wang, T., Yang, Y. and Jia, R., 2021. Improving cooperative game theory-based data valuation via data utility learning. arXiv preprint arXiv:2107.06336.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility","title":"Utility","text":"<pre><code>Utility(\n    model: SupervisedModel,\n    data: Dataset,\n    scorer: Optional[Union[str, Scorer]] = None,\n    *,\n    default_score: float = 0.0,\n    score_range: Tuple[float, float] = (-np.inf, np.inf),\n    catch_errors: bool = True,\n    show_warnings: bool = False,\n    cache_backend: Optional[CacheBackend] = None,\n    cached_func_options: Optional[CachedFuncConfig] = None,\n    clone_before_fit: bool = True\n)\n</code></pre> <p>Convenience wrapper with configurable memoization of the scoring function.</p> <p>An instance of <code>Utility</code> holds the triple of model, dataset and scoring function which determines the value of data points. This is used for the computation of all game-theoretic values like Shapley values and the Least Core.</p> <p>The Utility expect the model to fulfill the SupervisedModel interface i.e. to have <code>fit()</code>, <code>predict()</code>, and <code>score()</code> methods.</p> <p>When calling the utility, the model will be cloned if it is a Sci-Kit Learn model, otherwise a copy is created using copy.deepcopy</p> <p>Since evaluating the scoring function requires retraining the model and that can be time-consuming, this class wraps it and caches the results of each execution. Caching is available both locally and across nodes, but must always be enabled for your project first, see the documentation and the module documentation.</p> ATTRIBUTE DESCRIPTION <code>model</code> <p>The supervised model.</p> <p> TYPE: <code>SupervisedModel</code> </p> <code>data</code> <p>An object containing the split data.</p> <p> TYPE: <code>Dataset</code> </p> <code>scorer</code> <p>A scoring function. If None, the <code>score()</code> method of the model will be used. See score for ways to create and compose scorers, in particular how to set default values and ranges.</p> <p> TYPE: <code>Scorer</code> </p> PARAMETER  DESCRIPTION <code>model</code> <p>Any supervised model. Typical choices can be found in the [sci-kit learn documentation][https://scikit-learn.org/stable/supervised_learning.html].</p> <p> TYPE: <code>SupervisedModel</code> </p> <code>data</code> <p>Dataset or GroupedDataset instance.</p> <p> TYPE: <code>Dataset</code> </p> <code>scorer</code> <p>A scoring object. If None, the <code>score()</code> method of the model will be used. See score for ways to create and compose scorers, in particular how to set default values and ranges. For convenience, a string can be passed, which will be used to construct a Scorer.</p> <p> TYPE: <code>Optional[Union[str, Scorer]]</code> DEFAULT: <code>None</code> </p> <code>default_score</code> <p>As a convenience when no <code>scorer</code> object is passed (where a default value can be provided), this argument also allows to set the default score for models that have not been fit, e.g. when too little data is passed, or errors arise.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>score_range</code> <p>As with <code>default_score</code>, this is a convenience argument for when no <code>scorer</code> argument is provided, to set the numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation.</p> <p> TYPE: <code>Tuple[float, float]</code> DEFAULT: <code>(-inf, inf)</code> </p> <code>catch_errors</code> <p>set to <code>True</code> to catch the errors when <code>fit()</code> fails. This could happen in several steps of the pipeline, e.g. when too little training data is passed, which happens often during Shapley value calculations. When this happens, the <code>default_score</code> is returned as a score and computation continues.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>show_warnings</code> <p>Set to <code>False</code> to suppress warnings thrown by <code>fit()</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>cache_backend</code> <p>Optional instance of CacheBackend used to wrap the _utility method of the Utility instance. By default, this is set to None and that means that the utility evaluations will not be cached.</p> <p> TYPE: <code>Optional[CacheBackend]</code> DEFAULT: <code>None</code> </p> <code>cached_func_options</code> <p>Optional configuration object for cached utility evaluation.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> <code>clone_before_fit</code> <p>If <code>True</code>, the model will be cloned before calling <code>fit()</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Utility, DataUtilityLearning, Dataset\n&gt;&gt;&gt; from sklearn.linear_model import LinearRegression, LogisticRegression\n&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_iris(), random_state=16)\n&gt;&gt;&gt; u = Utility(LogisticRegression(random_state=16), dataset)\n&gt;&gt;&gt; u(dataset.indices)\n0.9\n</code></pre> <p>With caching enabled:</p> <pre><code>&gt;&gt;&gt; from pydvl.utils import Utility, DataUtilityLearning, Dataset\n&gt;&gt;&gt; from pydvl.utils.caching.memory import InMemoryCacheBackend\n&gt;&gt;&gt; from sklearn.linear_model import LinearRegression, LogisticRegression\n&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_iris(), random_state=16)\n&gt;&gt;&gt; cache_backend = InMemoryCacheBackend()\n&gt;&gt;&gt; u = Utility(LogisticRegression(random_state=16), dataset, cache_backend=cache_backend)\n&gt;&gt;&gt; u(dataset.indices)\n0.9\n</code></pre> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(\n    self,\n    model: SupervisedModel,\n    data: Dataset,\n    scorer: Optional[Union[str, Scorer]] = None,\n    *,\n    default_score: float = 0.0,\n    score_range: Tuple[float, float] = (-np.inf, np.inf),\n    catch_errors: bool = True,\n    show_warnings: bool = False,\n    cache_backend: Optional[CacheBackend] = None,\n    cached_func_options: Optional[CachedFuncConfig] = None,\n    clone_before_fit: bool = True,\n):\n    self.model = self._clone_model(model)\n    self.data = data\n    if isinstance(scorer, str):\n        scorer = Scorer(scorer, default=default_score, range=score_range)\n    self.scorer = check_scoring(self.model, scorer)\n    self.default_score = scorer.default if scorer is not None else default_score\n    # TODO: auto-fill from known scorers ?\n    self.score_range = scorer.range if scorer is not None else np.array(score_range)\n    self.clone_before_fit = clone_before_fit\n    self.catch_errors = catch_errors\n    self.show_warnings = show_warnings\n    self.cache = cache_backend\n    if cached_func_options is None:\n        cached_func_options = CachedFuncConfig()\n    # TODO: Find a better way to do this.\n    if cached_func_options.hash_prefix is None:\n        # FIX: This does not handle reusing the same across runs.\n        cached_func_options.hash_prefix = str(hash((model, data, scorer)))\n    self.cached_func_options = cached_func_options\n    self._initialize_utility_wrapper()\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility.cache_stats","title":"cache_stats  <code>property</code>","text":"<pre><code>cache_stats: Optional[CacheStats]\n</code></pre> <p>Cache statistics are gathered when cache is enabled. See CacheStats for all fields returned.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility.__call__","title":"__call__","text":"<pre><code>__call__(indices: Iterable[int]) -&gt; float\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>a subset of valid indices for the <code>x_train</code> attribute of Dataset.</p> <p> TYPE: <code>Iterable[int]</code> </p> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __call__(self, indices: Iterable[int]) -&gt; float:\n    \"\"\"\n    Args:\n        indices: a subset of valid indices for the\n            `x_train` attribute of [Dataset][pydvl.utils.dataset.Dataset].\n    \"\"\"\n    utility: float = self._utility_wrapper(frozenset(indices))\n    return utility\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.DataUtilityLearning","title":"DataUtilityLearning","text":"<pre><code>DataUtilityLearning(u: Utility, training_budget: int, model: SupervisedModel)\n</code></pre> <p>Implementation of Data Utility Learning (Wang et al., 2022)<sup>1</sup>.</p> <p>This object wraps a Utility and delegates calls to it, up until a given budget (number of iterations). Every tuple of input and output (a so-called utility sample) is stored. Once the budget is exhausted, <code>DataUtilityLearning</code> fits the given model to the utility samples. Subsequent calls will use the learned model to predict the utility instead of delegating.</p> PARAMETER  DESCRIPTION <code>u</code> <p>The Utility to learn.</p> <p> TYPE: <code>Utility</code> </p> <code>training_budget</code> <p>Number of utility samples to collect before fitting the given model.</p> <p> TYPE: <code>int</code> </p> <code>model</code> <p>A supervised regression model</p> <p> TYPE: <code>SupervisedModel</code> </p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Utility, DataUtilityLearning, Dataset\n&gt;&gt;&gt; from sklearn.linear_model import LinearRegression, LogisticRegression\n&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_iris())\n&gt;&gt;&gt; u = Utility(LogisticRegression(), dataset)\n&gt;&gt;&gt; wrapped_u = DataUtilityLearning(u, 3, LinearRegression())\n... # First 3 calls will be computed normally\n&gt;&gt;&gt; for i in range(3):\n...     _ = wrapped_u((i,))\n&gt;&gt;&gt; wrapped_u((1, 2, 3)) # Subsequent calls will be computed using the fit model for DUL\n0.0\n</code></pre> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(\n    self, u: Utility, training_budget: int, model: SupervisedModel\n) -&gt; None:\n    self.utility = u\n    self.training_budget = training_budget\n    self.model = model\n    self._current_iteration = 0\n    self._is_model_fit = False\n    self._utility_samples: Dict[FrozenSet, Tuple[NDArray[np.bool_], float]] = {}\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.DataUtilityLearning.data","title":"data  <code>property</code>","text":"<pre><code>data: Dataset\n</code></pre> <p>Returns the wrapped utility's Dataset.</p>"},{"location":"api/pydvl/utils/caching/","title":"Caching","text":""},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching","title":"pydvl.utils.caching","text":"<p>This module provides caching of functions.</p> <p>PyDVL can cache (memoize) the computation of the utility function and speed up some computations for data valuation.</p> <p>Warning</p> <p>Function evaluations are cached with a key based on the function's signature and code. This can lead to undesired cache hits, see Cache reuse.</p> <p>Remember not to reuse utility objects for different datasets.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--configuration","title":"Configuration","text":"<p>Caching is disabled by default but can be enabled easily, see Setting up the cache. When enabled, it will be added to any callable used to construct a Utility (done with the wrap method of CacheBackend). Depending on the nature of the utility you might want to enable the computation of a running average of function values, see Usage with stochastic functions. You can see all configuration options under CachedFuncConfig.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--supported-backends","title":"Supported Backends","text":"<p>pyDVL supports 3 different caching backends:</p> <ul> <li>InMemoryCacheBackend:   an in-memory cache backend that uses a dictionary to store and retrieve   cached values. This is used to share cached values between threads   in a single process.</li> <li>DiskCacheBackend:   a disk-based cache backend that uses pickled values written to and read from disk.   This is used to share cached values between processes in a single machine.</li> <li> <p>MemcachedCacheBackend:   a Memcached-based cache backend that uses pickled values written to   and read from a Memcached server. This is used to share cached values   between processes across multiple machines.</p> <p>Info</p> <p>This specific backend requires optional dependencies not installed by default. See Extra dependencies for more information.</p> </li> </ul>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--usage-with-stochastic-functions","title":"Usage with stochastic functions","text":"<p>In addition to standard memoization, the wrapped functions can compute running average and standard error of repeated evaluations for the same input. This can be useful for stochastic functions with high variance (e.g. model training for small sample sizes), but drastically reduces the speed benefits of memoization.</p> <p>This behaviour can be activated with the option allow_repeated_evaluations.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--cache-reuse","title":"Cache reuse","text":"<p>When working directly with CachedFunc,  it is essential to only cache pure functions. If they have any kind of state, either internal or external (e.g. a closure over some data that may change), then the cache will fail to notice this and the same value will be returned.</p> <p>When a function is wrapped with CachedFunc for memoization, its signature (input and output names) and code are used as a key for the cache.</p> <p>If you are running experiments with the same Utility but different datasets, this will lead to evaluations of the utility on new data returning old values because utilities only use sample indices as arguments (so there is no way to tell the difference between '1' for dataset A and '1' for dataset 2 from the point of view of the cache). One solution is to empty the cache between runs by calling the <code>clear</code> method of the cache backend instance, but the preferred one is to use a different Utility object for each dataset.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--unexpected-cache-misses","title":"Unexpected cache misses","text":"<p>Because all arguments to a function are used as part of the key for the cache, sometimes one must exclude some of them. For example, If a function is going to run across multiple processes and some reporting arguments are added (like a <code>job_id</code> for logging purposes), these will be part of the signature and make the functions distinct to the eyes of the cache. This can be avoided with the use of ignore_args option in the configuration.</p>"},{"location":"api/pydvl/utils/caching/base/","title":"Base","text":""},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base","title":"pydvl.utils.caching.base","text":""},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheStats","title":"CacheStats  <code>dataclass</code>","text":"<pre><code>CacheStats(\n    sets: int = 0,\n    misses: int = 0,\n    hits: int = 0,\n    timeouts: int = 0,\n    errors: int = 0,\n    reconnects: int = 0,\n)\n</code></pre> <p>Class used to store statistics gathered by cached functions.</p> ATTRIBUTE DESCRIPTION <code>sets</code> <p>Number of times a value was set in the cache.</p> <p> TYPE: <code>int</code> </p> <code>misses</code> <p>Number of times a value was not found in the cache.</p> <p> TYPE: <code>int</code> </p> <code>hits</code> <p>Number of times a value was found in the cache.</p> <p> TYPE: <code>int</code> </p> <code>timeouts</code> <p>Number of times a timeout occurred.</p> <p> TYPE: <code>int</code> </p> <code>errors</code> <p>Number of times an error occurred.</p> <p> TYPE: <code>int</code> </p> <code>reconnects</code> <p>Number of times the client reconnected to the server.</p> <p> TYPE: <code>int</code> </p>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheResult","title":"CacheResult  <code>dataclass</code>","text":"<pre><code>CacheResult(value: float, count: int = 1, variance: float = 0.0)\n</code></pre> <p>A class used to store the cached result of a computation as well as count and variance when using repeated evaluation.</p> ATTRIBUTE DESCRIPTION <code>value</code> <p>Cached value.</p> <p> TYPE: <code>float</code> </p> <code>count</code> <p>Number of times this value has been computed.</p> <p> TYPE: <code>int</code> </p> <code>variance</code> <p>Variance associated with the cached value.</p> <p> TYPE: <code>float</code> </p>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend","title":"CacheBackend","text":"<pre><code>CacheBackend()\n</code></pre> <p>             Bases: <code>ABC</code></p> <p>Abstract base class for cache backends.</p> <p>Defines interface for cache access including wrapping callables, getting/setting results, clearing cache, and combining cache keys.</p> ATTRIBUTE DESCRIPTION <code>stats</code> <p>Cache statistics tracker.</p> <p> </p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def __init__(self) -&gt; None:\n    self.stats = CacheStats()\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend.wrap","title":"wrap","text":"<pre><code>wrap(\n    func: Callable, *, config: Optional[CachedFuncConfig] = None\n) -&gt; CachedFunc\n</code></pre> <p>Wraps a function to cache its results.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The function to wrap.</p> <p> TYPE: <code>Callable</code> </p> <code>config</code> <p>Optional caching options for the wrapped function.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>CachedFunc</code> <p>The wrapped cached function.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def wrap(\n    self,\n    func: Callable,\n    *,\n    config: Optional[CachedFuncConfig] = None,\n) -&gt; \"CachedFunc\":\n    \"\"\"Wraps a function to cache its results.\n\n    Args:\n        func: The function to wrap.\n        config: Optional caching options for the wrapped function.\n\n    Returns:\n        The wrapped cached function.\n    \"\"\"\n    return CachedFunc(\n        func,\n        cache_backend=self,\n        config=config,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend.get","title":"get  <code>abstractmethod</code>","text":"<pre><code>get(key: str) -&gt; Optional[CacheResult]\n</code></pre> <p>Abstract method to retrieve a cached result.</p> <p>Implemented by subclasses.</p> PARAMETER  DESCRIPTION <code>key</code> <p>The cache key.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Optional[CacheResult]</code> <p>The cached result or None if not found.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>@abstractmethod\ndef get(self, key: str) -&gt; Optional[CacheResult]:\n    \"\"\"Abstract method to retrieve a cached result.\n\n    Implemented by subclasses.\n\n    Args:\n        key: The cache key.\n\n    Returns:\n        The cached result or None if not found.\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend.set","title":"set  <code>abstractmethod</code>","text":"<pre><code>set(key: str, value: CacheResult) -&gt; None\n</code></pre> <p>Abstract method to set a cached result.</p> <p>Implemented by subclasses.</p> PARAMETER  DESCRIPTION <code>key</code> <p>The cache key.</p> <p> TYPE: <code>str</code> </p> <code>value</code> <p>The result to cache.</p> <p> TYPE: <code>CacheResult</code> </p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>@abstractmethod\ndef set(self, key: str, value: CacheResult) -&gt; None:\n    \"\"\"Abstract method to set a cached result.\n\n    Implemented by subclasses.\n\n    Args:\n        key: The cache key.\n        value: The result to cache.\n    \"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend.clear","title":"clear  <code>abstractmethod</code>","text":"<pre><code>clear() -&gt; None\n</code></pre> <p>Abstract method to clear the entire cache.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>@abstractmethod\ndef clear(self) -&gt; None:\n    \"\"\"Abstract method to clear the entire cache.\"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CacheBackend.combine_hashes","title":"combine_hashes  <code>abstractmethod</code>","text":"<pre><code>combine_hashes(*args: str) -&gt; str\n</code></pre> <p>Abstract method to combine cache keys.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>@abstractmethod\ndef combine_hashes(self, *args: str) -&gt; str:\n    \"\"\"Abstract method to combine cache keys.\"\"\"\n    pass\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CachedFunc","title":"CachedFunc","text":"<pre><code>CachedFunc(\n    func: Callable[..., float],\n    *,\n    cache_backend: CacheBackend,\n    config: Optional[CachedFuncConfig] = None\n)\n</code></pre> <p>Caches callable function results with a provided cache backend.</p> <p>Wraps a callable function to cache its results using a provided an instance of a subclass of CacheBackend.</p> <p>This class is heavily inspired from that of joblib.memory.MemorizedFunc.</p> <p>This class caches calls to the wrapped callable by generating a hash key based on the wrapped callable's code, the arguments passed to it and the optional hash_prefix.</p> <p>Warning</p> <p>This class only works with hashable arguments to the wrapped callable.</p> PARAMETER  DESCRIPTION <code>func</code> <p>Callable to wrap.</p> <p> TYPE: <code>Callable[..., float]</code> </p> <code>cache_backend</code> <p>Instance of CacheBackendBase that handles setting and getting values.</p> <p> TYPE: <code>CacheBackend</code> </p> <code>config</code> <p>Configuration for wrapped function.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def __init__(\n    self,\n    func: Callable[..., float],\n    *,\n    cache_backend: CacheBackend,\n    config: Optional[CachedFuncConfig] = None,\n) -&gt; None:\n    self.func = func\n    self.cache_backend = cache_backend\n    if config is None:\n        config = CachedFuncConfig()\n    self.config = config\n\n    self.__doc__ = f\"A wrapper around {func.__name__}() with caching enabled.\\n\" + (\n        CachedFunc.__doc__ or \"\"\n    )\n    self.__name__ = f\"cached_{func.__name__}\"\n    path = list(reversed(func.__qualname__.split(\".\")))\n    patched = [f\"cached_{path[0]}\"] + path[1:]\n    self.__qualname__ = \".\".join(reversed(patched))\n</code></pre>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CachedFunc.stats","title":"stats  <code>property</code>","text":"<pre><code>stats: CacheStats\n</code></pre> <p>Cache backend statistics.</p>"},{"location":"api/pydvl/utils/caching/base/#pydvl.utils.caching.base.CachedFunc.__call__","title":"__call__","text":"<pre><code>__call__(*args, **kwargs) -&gt; float\n</code></pre> <p>Call the wrapped cached function.</p> <p>Executes the wrapped function, caching and returning the result.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def __call__(self, *args, **kwargs) -&gt; float:\n    \"\"\"Call the wrapped cached function.\n\n    Executes the wrapped function, caching and returning the result.\n    \"\"\"\n    return self._cached_call(args, kwargs)\n</code></pre>"},{"location":"api/pydvl/utils/caching/config/","title":"Config","text":""},{"location":"api/pydvl/utils/caching/config/#pydvl.utils.caching.config","title":"pydvl.utils.caching.config","text":""},{"location":"api/pydvl/utils/caching/config/#pydvl.utils.caching.config.CachedFuncConfig","title":"CachedFuncConfig  <code>dataclass</code>","text":"<pre><code>CachedFuncConfig(\n    hash_prefix: Optional[str] = None,\n    ignore_args: Collection[str] = list(),\n    time_threshold: float = 0.3,\n    allow_repeated_evaluations: bool = False,\n    rtol_stderr: float = 0.1,\n    min_repetitions: int = 3,\n)\n</code></pre> <p>Configuration for cached functions and methods, providing memoization of function calls.</p> <p>Instances of this class are typically used as arguments for the construction of a Utility.</p> PARAMETER  DESCRIPTION <code>hash_prefix</code> <p>Optional string prefix that be prepended to the cache key. This can be provided in order to guarantee cache reuse across runs.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ignore_args</code> <p>Do not take these keyword arguments into account when hashing the wrapped function for usage as key. This allows sharing the cache among different jobs for the same experiment run if the callable happens to have \"nuisance\" parameters like <code>job_id</code> which do not affect the result of the computation.</p> <p> TYPE: <code>Collection[str]</code> DEFAULT: <code>list()</code> </p> <code>time_threshold</code> <p>Computations taking less time than this many seconds are not cached. A value of 0 means that it will always cache results.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.3</code> </p> <code>allow_repeated_evaluations</code> <p>If <code>True</code>, repeated calls to a function with the same arguments will be allowed and outputs averaged until the running standard deviation of the mean stabilizes below <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>rtol_stderr</code> <p>relative tolerance for repeated evaluations. More precisely, memcached() will stop evaluating the function once the standard deviation of the mean is smaller than <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.1</code> </p> <code>min_repetitions</code> <p>minimum number of times that a function evaluation on the same arguments is repeated before returning cached values. Useful for stochastic functions only. If the model training is very noisy, set this number to higher values to reduce variance.</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p>"},{"location":"api/pydvl/utils/caching/disk/","title":"Disk","text":""},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk","title":"pydvl.utils.caching.disk","text":""},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend","title":"DiskCacheBackend","text":"<pre><code>DiskCacheBackend(cache_dir: Optional[Union[PathLike, str]] = None)\n</code></pre> <p>             Bases: <code>CacheBackend</code></p> <p>Disk cache backend that stores results in files.</p> <p>Implements the CacheBackend interface for a disk-based cache. Stores cache entries as pickled files on disk, keyed by cache key. This allows sharing evaluations across processes in a single node/computer.</p> PARAMETER  DESCRIPTION <code>cache_dir</code> <p>Base directory for cache storage.</p> <p> TYPE: <code>Optional[Union[PathLike, str]]</code> DEFAULT: <code>None</code> </p> ATTRIBUTE DESCRIPTION <code>cache_dir</code> <p>Base directory for cache storage.</p> <p> </p> Example <p>Basic usage: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.disk import DiskCacheBackend\n&gt;&gt;&gt; cache_backend = DiskCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; cache_backend.set(\"key\", value)\n&gt;&gt;&gt; cache_backend.get(\"key\")\n42\n</code></pre></p> <p>Callable wrapping: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.disk import DiskCacheBackend\n&gt;&gt;&gt; cache_backend = DiskCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; def foo(x: int):\n...     return x + 1\n...\n&gt;&gt;&gt; wrapped_foo = cache_backend.wrap(foo)\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n0\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n1\n</code></pre></p> PARAMETER  DESCRIPTION <code>cache_dir</code> <p>Base directory for cache storage. If not provided, this defaults to a newly created temporary directory.</p> <p> TYPE: <code>Optional[Union[PathLike, str]]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/utils/caching/disk.py</code> <pre><code>def __init__(\n    self,\n    cache_dir: Optional[Union[os.PathLike, str]] = None,\n) -&gt; None:\n    \"\"\"Initialize the disk cache backend.\n\n    Args:\n        cache_dir: Base directory for cache storage.\n            If not provided, this defaults to a newly created\n            temporary directory.\n    \"\"\"\n    super().__init__()\n    if cache_dir is None:\n        cache_dir = tempfile.mkdtemp(prefix=\"pydvl\")\n    self.cache_dir = Path(cache_dir)\n    self.cache_dir.mkdir(exist_ok=True, parents=True)\n</code></pre>"},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend.wrap","title":"wrap","text":"<pre><code>wrap(\n    func: Callable, *, config: Optional[CachedFuncConfig] = None\n) -&gt; CachedFunc\n</code></pre> <p>Wraps a function to cache its results.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The function to wrap.</p> <p> TYPE: <code>Callable</code> </p> <code>config</code> <p>Optional caching options for the wrapped function.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>CachedFunc</code> <p>The wrapped cached function.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def wrap(\n    self,\n    func: Callable,\n    *,\n    config: Optional[CachedFuncConfig] = None,\n) -&gt; \"CachedFunc\":\n    \"\"\"Wraps a function to cache its results.\n\n    Args:\n        func: The function to wrap.\n        config: Optional caching options for the wrapped function.\n\n    Returns:\n        The wrapped cached function.\n    \"\"\"\n    return CachedFunc(\n        func,\n        cache_backend=self,\n        config=config,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend.get","title":"get","text":"<pre><code>get(key: str) -&gt; Optional[Any]\n</code></pre> <p>Get a value from the cache.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Optional[Any]</code> <p>Cached value or None if not found.</p> Source code in <code>src/pydvl/utils/caching/disk.py</code> <pre><code>def get(self, key: str) -&gt; Optional[Any]:\n    \"\"\"Get a value from the cache.\n\n    Args:\n        key: Cache key.\n\n    Returns:\n        Cached value or None if not found.\n    \"\"\"\n    cache_file = self.cache_dir / key\n    if not cache_file.exists():\n        self.stats.misses += 1\n        return None\n    self.stats.hits += 1\n    with cache_file.open(\"rb\") as f:\n        return cloudpickle.load(f)\n</code></pre>"},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend.set","title":"set","text":"<pre><code>set(key: str, value: Any) -&gt; None\n</code></pre> <p>Set a value in the cache.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> <code>value</code> <p>Value to cache.</p> <p> TYPE: <code>Any</code> </p> Source code in <code>src/pydvl/utils/caching/disk.py</code> <pre><code>def set(self, key: str, value: Any) -&gt; None:\n    \"\"\"Set a value in the cache.\n\n    Args:\n        key: Cache key.\n        value: Value to cache.\n    \"\"\"\n    cache_file = self.cache_dir / key\n    self.stats.sets += 1\n    with cache_file.open(\"wb\") as f:\n        cloudpickle.dump(value, f, protocol=PICKLE_VERSION)\n</code></pre>"},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend.clear","title":"clear","text":"<pre><code>clear() -&gt; None\n</code></pre> <p>Deletes cache directory and recreates it.</p> Source code in <code>src/pydvl/utils/caching/disk.py</code> <pre><code>def clear(self) -&gt; None:\n    \"\"\"Deletes cache directory and recreates it.\"\"\"\n    shutil.rmtree(self.cache_dir)\n    self.cache_dir.mkdir(exist_ok=True, parents=True)\n</code></pre>"},{"location":"api/pydvl/utils/caching/disk/#pydvl.utils.caching.disk.DiskCacheBackend.combine_hashes","title":"combine_hashes","text":"<pre><code>combine_hashes(*args: str) -&gt; str\n</code></pre> <p>Join cache key components.</p> Source code in <code>src/pydvl/utils/caching/disk.py</code> <pre><code>def combine_hashes(self, *args: str) -&gt; str:\n    \"\"\"Join cache key components.\"\"\"\n    return os.pathsep.join(args)\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/","title":"Memcached","text":""},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached","title":"pydvl.utils.caching.memcached","text":""},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedClientConfig","title":"MemcachedClientConfig  <code>dataclass</code>","text":"<pre><code>MemcachedClientConfig(\n    server: Tuple[str, int] = (\"localhost\", 11211),\n    connect_timeout: float = 1.0,\n    timeout: float = 1.0,\n    no_delay: bool = True,\n    serde: PickleSerde = PickleSerde(pickle_version=PICKLE_VERSION),\n)\n</code></pre> <p>Configuration of the memcached client.</p> PARAMETER  DESCRIPTION <code>server</code> <p>A tuple of (IP|domain name, port).</p> <p> TYPE: <code>Tuple[str, int]</code> DEFAULT: <code>('localhost', 11211)</code> </p> <code>connect_timeout</code> <p>How many seconds to wait before raising <code>ConnectionRefusedError</code> on failure to connect.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>timeout</code> <p>Duration in seconds to wait for send or recv calls on the socket connected to memcached.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>no_delay</code> <p>If True, set the <code>TCP_NODELAY</code> flag, which may help with performance in some cases.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>serde</code> <p>Serializer / Deserializer (\"serde\"). The default <code>PickleSerde</code> should work in most cases. See pymemcache.client.base.Client for details.</p> <p> TYPE: <code>PickleSerde</code> DEFAULT: <code>PickleSerde(pickle_version=PICKLE_VERSION)</code> </p>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend","title":"MemcachedCacheBackend","text":"<pre><code>MemcachedCacheBackend(config: MemcachedClientConfig = MemcachedClientConfig())\n</code></pre> <p>             Bases: <code>CacheBackend</code></p> <p>Memcached cache backend for the distributed caching of functions.</p> <p>Implements the CacheBackend interface for a memcached based cache. This allows sharing evaluations across processes and nodes in a cluster. You can run memcached as a service, locally or remotely, see the caching documentation.</p> PARAMETER  DESCRIPTION <code>config</code> <p>Memcached client configuration.</p> <p> TYPE: <code>MemcachedClientConfig</code> DEFAULT: <code>MemcachedClientConfig()</code> </p> ATTRIBUTE DESCRIPTION <code>config</code> <p>Memcached client configuration.</p> <p> </p> <code>client</code> <p>Memcached client instance.</p> <p> </p> Example <p>Basic usage: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.memcached import MemcachedCacheBackend\n&gt;&gt;&gt; cache_backend = MemcachedCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; cache_backend.set(\"key\", value)\n&gt;&gt;&gt; cache_backend.get(\"key\")\n42\n</code></pre></p> <p>Callable wrapping: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.memcached import MemcachedCacheBackend\n&gt;&gt;&gt; cache_backend = MemcachedCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; def foo(x: int):\n...     return x + 1\n...\n&gt;&gt;&gt; wrapped_foo = cache_backend.wrap(foo)\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n0\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n1\n</code></pre></p> PARAMETER  DESCRIPTION <code>config</code> <p>Memcached client configuration.</p> <p> TYPE: <code>MemcachedClientConfig</code> DEFAULT: <code>MemcachedClientConfig()</code> </p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def __init__(self, config: MemcachedClientConfig = MemcachedClientConfig()) -&gt; None:\n    \"\"\"Initialize memcached cache backend.\n\n    Args:\n        config: Memcached client configuration.\n    \"\"\"\n\n    super().__init__()\n    self.config = config\n    self.client = self._connect(self.config)\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.wrap","title":"wrap","text":"<pre><code>wrap(\n    func: Callable, *, config: Optional[CachedFuncConfig] = None\n) -&gt; CachedFunc\n</code></pre> <p>Wraps a function to cache its results.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The function to wrap.</p> <p> TYPE: <code>Callable</code> </p> <code>config</code> <p>Optional caching options for the wrapped function.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>CachedFunc</code> <p>The wrapped cached function.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def wrap(\n    self,\n    func: Callable,\n    *,\n    config: Optional[CachedFuncConfig] = None,\n) -&gt; \"CachedFunc\":\n    \"\"\"Wraps a function to cache its results.\n\n    Args:\n        func: The function to wrap.\n        config: Optional caching options for the wrapped function.\n\n    Returns:\n        The wrapped cached function.\n    \"\"\"\n    return CachedFunc(\n        func,\n        cache_backend=self,\n        config=config,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.get","title":"get","text":"<pre><code>get(key: str) -&gt; Optional[Any]\n</code></pre> <p>Get value from memcached.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Optional[Any]</code> <p>Cached value or None if not found or client disconnected.</p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def get(self, key: str) -&gt; Optional[Any]:\n    \"\"\"Get value from memcached.\n\n    Args:\n        key: Cache key.\n\n    Returns:\n        Cached value or None if not found or client disconnected.\n    \"\"\"\n    result = None\n    try:\n        result = self.client.get(key)\n    except socket.timeout as e:\n        self.stats.timeouts += 1\n        warnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\n    except OSError as e:\n        self.stats.errors += 1\n        warnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\n    except AttributeError as e:\n        # FIXME: this depends on _recv() failing on invalid sockets\n        # See pymemcache.base.py,\n        self.stats.reconnects += 1\n        warnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\n        self.client = self._connect(self.config)\n    if result is None:\n        self.stats.misses += 1\n    else:\n        self.stats.hits += 1\n    return result\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.set","title":"set","text":"<pre><code>set(key: str, value: Any) -&gt; None\n</code></pre> <p>Set value in memcached.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> <code>value</code> <p>Value to cache.</p> <p> TYPE: <code>Any</code> </p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def set(self, key: str, value: Any) -&gt; None:\n    \"\"\"Set value in memcached.\n\n    Args:\n        key: Cache key.\n        value: Value to cache.\n    \"\"\"\n    self.client.set(key, value, noreply=True)\n    self.stats.sets += 1\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.clear","title":"clear","text":"<pre><code>clear() -&gt; None\n</code></pre> <p>Flush all values from memcached.</p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def clear(self) -&gt; None:\n    \"\"\"Flush all values from memcached.\"\"\"\n    self.client.flush_all(noreply=True)\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.combine_hashes","title":"combine_hashes","text":"<pre><code>combine_hashes(*args: str) -&gt; str\n</code></pre> <p>Join cache key components for Memcached.</p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def combine_hashes(self, *args: str) -&gt; str:\n    \"\"\"Join cache key components for Memcached.\"\"\"\n    return \":\".join(args)\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.__getstate__","title":"__getstate__","text":"<pre><code>__getstate__() -&gt; Dict\n</code></pre> <p>Enables pickling after a socket has been opened to the memcached server, by removing the client from the stored data.</p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def __getstate__(self) -&gt; Dict:\n    \"\"\"Enables pickling after a socket has been opened to the\n    memcached server, by removing the client from the stored\n    data.\"\"\"\n    odict = self.__dict__.copy()\n    del odict[\"client\"]\n    return odict\n</code></pre>"},{"location":"api/pydvl/utils/caching/memcached/#pydvl.utils.caching.memcached.MemcachedCacheBackend.__setstate__","title":"__setstate__","text":"<pre><code>__setstate__(d: Dict)\n</code></pre> <p>Restores a client connection after loading from a pickle.</p> Source code in <code>src/pydvl/utils/caching/memcached.py</code> <pre><code>def __setstate__(self, d: Dict):\n    \"\"\"Restores a client connection after loading from a pickle.\"\"\"\n    self.config = d[\"config\"]\n    self.stats = d[\"stats\"]\n    self.client = self._connect(self.config)\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/","title":"Memory","text":""},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory","title":"pydvl.utils.caching.memory","text":""},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend","title":"InMemoryCacheBackend","text":"<pre><code>InMemoryCacheBackend()\n</code></pre> <p>             Bases: <code>CacheBackend</code></p> <p>In-memory cache backend that stores results in a dictionary.</p> <p>Implements the CacheBackend interface for an in-memory-based cache. Stores cache entries as values in a dictionary, keyed by cache key. This allows sharing evaluations across threads in a single process.</p> <p>The implementation is not thread-safe.</p> ATTRIBUTE DESCRIPTION <code>cached_values</code> <p>Dictionary used to store cached values.</p> <p> TYPE: <code>Dict[str, Any]</code> </p> Example <p>Basic usage: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.memory import InMemoryCacheBackend\n&gt;&gt;&gt; cache_backend = InMemoryCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; cache_backend.set(\"key\", value)\n&gt;&gt;&gt; cache_backend.get(\"key\")\n42\n</code></pre></p> <p>Callable wrapping: <pre><code>&gt;&gt;&gt; from pydvl.utils.caching.memory import InMemoryCacheBackend\n&gt;&gt;&gt; cache_backend = InMemoryCacheBackend()\n&gt;&gt;&gt; cache_backend.clear()\n&gt;&gt;&gt; value = 42\n&gt;&gt;&gt; def foo(x: int):\n...     return x + 1\n...\n&gt;&gt;&gt; wrapped_foo = cache_backend.wrap(foo)\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n0\n&gt;&gt;&gt; wrapped_foo(value)\n43\n&gt;&gt;&gt; wrapped_foo.stats.misses\n1\n&gt;&gt;&gt; wrapped_foo.stats.hits\n1\n</code></pre></p> Source code in <code>src/pydvl/utils/caching/memory.py</code> <pre><code>def __init__(self) -&gt; None:\n    \"\"\"Initialize the in-memory cache backend.\"\"\"\n    super().__init__()\n    self.cached_values: Dict[str, Any] = {}\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend.wrap","title":"wrap","text":"<pre><code>wrap(\n    func: Callable, *, config: Optional[CachedFuncConfig] = None\n) -&gt; CachedFunc\n</code></pre> <p>Wraps a function to cache its results.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The function to wrap.</p> <p> TYPE: <code>Callable</code> </p> <code>config</code> <p>Optional caching options for the wrapped function.</p> <p> TYPE: <code>Optional[CachedFuncConfig]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>CachedFunc</code> <p>The wrapped cached function.</p> Source code in <code>src/pydvl/utils/caching/base.py</code> <pre><code>def wrap(\n    self,\n    func: Callable,\n    *,\n    config: Optional[CachedFuncConfig] = None,\n) -&gt; \"CachedFunc\":\n    \"\"\"Wraps a function to cache its results.\n\n    Args:\n        func: The function to wrap.\n        config: Optional caching options for the wrapped function.\n\n    Returns:\n        The wrapped cached function.\n    \"\"\"\n    return CachedFunc(\n        func,\n        cache_backend=self,\n        config=config,\n    )\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend.get","title":"get","text":"<pre><code>get(key: str) -&gt; Optional[Any]\n</code></pre> <p>Get a value from the cache.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Optional[Any]</code> <p>Cached value or None if not found.</p> Source code in <code>src/pydvl/utils/caching/memory.py</code> <pre><code>def get(self, key: str) -&gt; Optional[Any]:\n    \"\"\"Get a value from the cache.\n\n    Args:\n        key: Cache key.\n\n    Returns:\n        Cached value or None if not found.\n    \"\"\"\n    value = self.cached_values.get(key, None)\n    if value is not None:\n        self.stats.hits += 1\n    else:\n        self.stats.misses += 1\n    return value\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend.set","title":"set","text":"<pre><code>set(key: str, value: Any) -&gt; None\n</code></pre> <p>Set a value in the cache.</p> PARAMETER  DESCRIPTION <code>key</code> <p>Cache key.</p> <p> TYPE: <code>str</code> </p> <code>value</code> <p>Value to cache.</p> <p> TYPE: <code>Any</code> </p> Source code in <code>src/pydvl/utils/caching/memory.py</code> <pre><code>def set(self, key: str, value: Any) -&gt; None:\n    \"\"\"Set a value in the cache.\n\n    Args:\n        key: Cache key.\n        value: Value to cache.\n    \"\"\"\n    self.cached_values[key] = value\n    self.stats.sets += 1\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend.clear","title":"clear","text":"<pre><code>clear() -&gt; None\n</code></pre> <p>Deletes cache dictionary and recreates it.</p> Source code in <code>src/pydvl/utils/caching/memory.py</code> <pre><code>def clear(self) -&gt; None:\n    \"\"\"Deletes cache dictionary and recreates it.\"\"\"\n    del self.cached_values\n    self.cached_values = {}\n</code></pre>"},{"location":"api/pydvl/utils/caching/memory/#pydvl.utils.caching.memory.InMemoryCacheBackend.combine_hashes","title":"combine_hashes","text":"<pre><code>combine_hashes(*args: str) -&gt; str\n</code></pre> <p>Join cache key components.</p> Source code in <code>src/pydvl/utils/caching/memory.py</code> <pre><code>def combine_hashes(self, *args: str) -&gt; str:\n    \"\"\"Join cache key components.\"\"\"\n    return os.pathsep.join(args)\n</code></pre>"},{"location":"api/pydvl/value/","title":"Value","text":""},{"location":"api/pydvl/value/#pydvl.value","title":"pydvl.value","text":"<p>This module implements algorithms for the exact and approximate computation of values and semi-values.</p> <p>See Data valuation for an introduction to the concepts and methods implemented here.</p>"},{"location":"api/pydvl/value/games/","title":"Games","text":""},{"location":"api/pydvl/value/games/#pydvl.value.games","title":"pydvl.value.games","text":"<p>This module provides several predefined games and, depending on the game, the corresponding Shapley values, Least Core values or both of them, for benchmarking purposes.</p>"},{"location":"api/pydvl/value/games/#pydvl.value.games--references","title":"References","text":"<ol> <li> <p>Castro, J., G\u00f3mez, D. and Tejada,   J., 2009. Polynomial calculation of the Shapley value based on   sampling.   Computers &amp; Operations Research, 36(5), pp.1726-1730.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset","title":"DummyGameDataset","text":"<pre><code>DummyGameDataset(n_players: int, description: Optional[str] = None)\n</code></pre> <p>             Bases: <code>Dataset</code></p> <p>Dummy game dataset.</p> <p>Initializes a dummy game dataset with n_players and an optional description.</p> <p>This class is used internally inside the Game class.</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> </p> <code>description</code> <p>Optional description of the dataset.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int, description: Optional[str] = None) -&gt; None:\n    x = np.arange(0, n_players, 1).reshape(-1, 1)\n    nil = np.zeros_like(x)\n    super().__init__(\n        x,\n        nil.copy(),\n        nil.copy(),\n        nil.copy(),\n        feature_names=[\"x\"],\n        target_names=[\"y\"],\n        description=description,\n    )\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.indices","title":"indices  <code>property</code>","text":"<pre><code>indices: NDArray[int_]\n</code></pre> <p>Index of positions in data.x_train.</p> <p>Contiguous integers from 0 to len(Dataset).</p>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.data_names","title":"data_names  <code>property</code>","text":"<pre><code>data_names: NDArray[object_]\n</code></pre> <p>Names of each individual datapoint.</p> <p>Used for reporting Shapley values.</p>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.dim","title":"dim  <code>property</code>","text":"<pre><code>dim: int\n</code></pre> <p>Returns the number of dimensions of a sample.</p>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.get_training_data","title":"get_training_data","text":"<pre><code>get_training_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Given a set of indices, returns the training data that refer to those indices.</p> <p>This is used mainly by Utility to retrieve subsets of the data from indices. It is typically not needed in algorithms.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices that will be used to select points from the training data. If <code>None</code>, the entire training data will be returned.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>If <code>indices</code> is not <code>None</code>, the selected x and y arrays from the training data. Otherwise, the entire dataset.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_training_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Given a set of indices, returns the training data that refer to those\n    indices.\n\n    This is used mainly by [Utility][pydvl.utils.utility.Utility] to retrieve\n    subsets of the data from indices. It is typically **not needed in\n    algorithms**.\n\n    Args:\n        indices: Optional indices that will be used to select points from\n            the training data. If `None`, the entire training data will be\n            returned.\n\n    Returns:\n        If `indices` is not `None`, the selected x and y arrays from the\n            training data. Otherwise, the entire dataset.\n    \"\"\"\n    if indices is None:\n        return self.x_train, self.y_train\n    x = self.x_train[indices]\n    y = self.y_train[indices]\n    return x, y\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.from_sklearn","title":"from_sklearn  <code>classmethod</code>","text":"<pre><code>from_sklearn(\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs\n) -&gt; Dataset\n</code></pre> <p>Constructs a Dataset object from a sklearn.utils.Bunch, as returned by the <code>load_*</code> functions in scikit-learn toy datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import load_boston\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n</code></pre> PARAMETER  DESCRIPTION <code>data</code> <p>scikit-learn Bunch object. The following attributes are supported:</p> <ul> <li><code>data</code>: covariates.</li> <li><code>target</code>: target variables (labels).</li> <li><code>feature_names</code> (optional): the feature names.</li> <li><code>target_names</code> (optional): the target names.</li> <li><code>DESCR</code> (optional): a description.</li> </ul> <p> TYPE: <code>Bunch</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the target variable as labels. Read more in scikit-learn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>is_multi_output</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the sklearn dataset</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_sklearn(\n    cls,\n    data: Bunch,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs,\n) -&gt; \"Dataset\":\n    \"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from a\n    [sklearn.utils.Bunch][], as returned by the `load_*`\n    functions in [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html).\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import load_boston\n        &gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n        ```\n\n    Args:\n        data: scikit-learn Bunch object. The following attributes are supported:\n\n            - `data`: covariates.\n            - `target`: target variables (labels).\n            - `feature_names` (**optional**): the feature names.\n            - `target_names` (**optional**): the target names.\n            - `DESCR` (**optional**): a description.\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the target variable as labels. Read more in\n            [scikit-learn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `is_multi_output`.\n\n    Returns:\n        Object with the sklearn dataset\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\n    x_train, x_test, y_train, y_test = train_test_split(\n        data.data,\n        data.target,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=data.target if stratify_by_target else None,\n    )\n    return cls(\n        x_train,\n        y_train,\n        x_test,\n        y_test,\n        feature_names=data.get(\"feature_names\"),\n        target_names=data.get(\"target_names\"),\n        description=data.get(\"DESCR\"),\n        **kwargs,\n    )\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.from_arrays","title":"from_arrays  <code>classmethod</code>","text":"<pre><code>from_arrays(\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs\n) -&gt; Dataset\n</code></pre> <p>Constructs a Dataset object from X and y numpy arrays  as returned by the <code>make_*</code> functions in sklearn generated datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import make_regression\n&gt;&gt;&gt; X, y = make_regression()\n&gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n</code></pre> PARAMETER  DESCRIPTION <code>X</code> <p>numpy array of shape (n_samples, n_features)</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>numpy array of shape (n_samples,)</p> <p> TYPE: <code>NDArray</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the y variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>feature_names</code> or <code>target_names</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the passed X and y arrays split across training and test sets.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_arrays(\n    cls,\n    X: NDArray,\n    y: NDArray,\n    train_size: float = 0.8,\n    random_state: Optional[int] = None,\n    stratify_by_target: bool = False,\n    **kwargs,\n) -&gt; \"Dataset\":\n    \"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from X and y numpy arrays  as\n    returned by the `make_*` functions in [sklearn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).\n\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import make_regression\n        &gt;&gt;&gt; X, y = make_regression()\n        &gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n        ```\n\n    Args:\n        X: numpy array of shape (n_samples, n_features)\n        y: numpy array of shape (n_samples,)\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified fashion,\n            using the y variable as labels. Read more in [sklearn's user\n            guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `feature_names`\n            or `target_names`.\n\n    Returns:\n        Object with the passed X and y arrays split across training and test sets.\n\n    !!! tip \"New in version 0.4.0\"\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\n    x_train, x_test, y_train, y_test = train_test_split(\n        X,\n        y,\n        train_size=train_size,\n        random_state=random_state,\n        stratify=y if stratify_by_target else None,\n    )\n    return cls(x_train, y_train, x_test, y_test, **kwargs)\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyGameDataset.get_test_data","title":"get_test_data","text":"<pre><code>get_test_data(\n    indices: Optional[Iterable[int]] = None,\n) -&gt; Tuple[NDArray, NDArray]\n</code></pre> <p>Returns the subsets of the train set instead of the test set.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>Indices into the training data.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>Subset of the train data.</p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def get_test_data(\n    self, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n    \"\"\"Returns the subsets of the train set instead of the test set.\n\n    Args:\n        indices: Indices into the training data.\n\n    Returns:\n        Subset of the train data.\n    \"\"\"\n    if indices is None:\n        return self.x_train, self.y_train\n    x = self.x_train[indices]\n    y = self.y_train[indices]\n    return x, y\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.DummyModel","title":"DummyModel","text":"<pre><code>DummyModel()\n</code></pre> <p>             Bases: <code>SupervisedModel</code></p> <p>Dummy model class.</p> <p>A dummy supervised model used for testing purposes only.</p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self) -&gt; None:\n    pass\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.Game","title":"Game","text":"<pre><code>Game(\n    n_players: int,\n    score_range: Tuple[float, float] = (-np.inf, np.inf),\n    description: Optional[str] = None,\n)\n</code></pre> <p>             Bases: <code>ABC</code></p> <p>Base class for games</p> <p>Any Game subclass has to implement the abstract <code>_score</code> method to assign a score to each coalition/subset and at least one of <code>shapley_values</code>, <code>least_core_values</code>.</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> </p> <code>score_range</code> <p>Minimum and maximum values of the <code>_score</code> method.</p> <p> TYPE: <code>Tuple[float, float]</code> DEFAULT: <code>(-inf, inf)</code> </p> <code>description</code> <p>Optional string description of the dummy dataset that will be created.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> ATTRIBUTE DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> </p> <code>data</code> <p>Dummy dataset object.</p> <p> </p> <code>u</code> <p>Utility object with a dummy model and dataset.</p> <p> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(\n    self,\n    n_players: int,\n    score_range: Tuple[float, float] = (-np.inf, np.inf),\n    description: Optional[str] = None,\n):\n    self.n_players = n_players\n    self.data = DummyGameDataset(self.n_players, description)\n    self.u = Utility(\n        DummyModel(),\n        self.data,\n        scorer=Scorer(self._score, range=score_range),\n        catch_errors=False,\n        show_warnings=True,\n    )\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.SymmetricVotingGame","title":"SymmetricVotingGame","text":"<pre><code>SymmetricVotingGame(n_players: int)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>A symmetric voting game defined in (Castro et al., 2009)<sup>1</sup> Section 4.1</p> <p>For this game the utility of a coalition is 1 if its cardinality is greater than num_samples/2, or 0 otherwise.</p> \\[{ v(S) = \\left\\{\\begin{array}{ll} 1, &amp; \\text{ if} \\quad \\mid S \\mid &gt; \\frac{N}{2} \\\\ 0, &amp; \\text{ otherwise} \\end{array}\\right. }\\] PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int) -&gt; None:\n    if n_players % 2 != 0:\n        raise ValueError(\"n_players must be an even number.\")\n    description = \"Dummy data for the symmetric voting game in Castro et al. 2009\"\n    super().__init__(\n        n_players,\n        score_range=(0, 1),\n        description=description,\n    )\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.AsymmetricVotingGame","title":"AsymmetricVotingGame","text":"<pre><code>AsymmetricVotingGame(n_players: int = 51)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>An asymmetric voting game defined in (Castro et al., 2009)<sup>1</sup> Section 4.2.</p> <p>For this game the player set is \\(N = \\{1,\\dots,51\\}\\) and the utility of a coalition is given by:</p> \\[{ v(S) = \\left\\{\\begin{array}{ll} 1, &amp; \\text{ if} \\quad \\sum\\limits_{i \\in S} w_i &gt; \\sum\\limits_{j \\in N}\\frac{w_j}{2} \\\\ 0, &amp; \\text{ otherwise} \\end{array}\\right. }\\] <p>where \\(w = [w_1,\\dots, w_{51}]\\) is a list of weights associated with each player.</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> DEFAULT: <code>51</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int = 51) -&gt; None:\n    if n_players != 51:\n        raise ValueError(\n            f\"{self.__class__.__name__} only supports n_players=51 but got {n_players=}.\"\n        )\n    description = \"Dummy data for the asymmetric voting game in Castro et al. 2009\"\n    super().__init__(\n        n_players,\n        score_range=(0, 1),\n        description=description,\n    )\n\n    ranges = [\n        range(0, 1),\n        range(1, 2),\n        range(2, 3),\n        range(3, 5),\n        range(5, 6),\n        range(6, 7),\n        range(7, 9),\n        range(9, 10),\n        range(10, 12),\n        range(12, 15),\n        range(15, 16),\n        range(16, 20),\n        range(20, 24),\n        range(24, 26),\n        range(26, 30),\n        range(30, 34),\n        range(34, 35),\n        range(35, 44),\n        range(44, 51),\n    ]\n\n    ranges_weights = [\n        45,\n        41,\n        27,\n        26,\n        25,\n        21,\n        17,\n        14,\n        13,\n        12,\n        11,\n        10,\n        9,\n        8,\n        7,\n        6,\n        5,\n        4,\n        3,\n    ]\n    ranges_values = [\n        \"0.08831\",\n        \"0.07973\",\n        \"0.05096\",\n        \"0.04898\",\n        \"0.047\",\n        \"0.03917\",\n        \"0.03147\",\n        \"0.02577\",\n        \"0.02388\",\n        \"0.022\",\n        \"0.02013\",\n        \"0.01827\",\n        \"0.01641\",\n        \"0.01456\",\n        \"0.01272\",\n        \"0.01088\",\n        \"0.009053\",\n        \"0.00723\",\n        \"0.005412\",\n    ]\n\n    self.weight_table = np.zeros(self.n_players)\n    exact_values = np.zeros(self.n_players)\n    for r, w, v in zip(ranges, ranges_weights, ranges_values):\n        self.weight_table[r] = w\n        exact_values[r] = v\n\n    self.exact_values = exact_values\n    self.threshold = np.sum(self.weight_table) / 2\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.ShoesGame","title":"ShoesGame","text":"<pre><code>ShoesGame(left: int, right: int)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>A shoes game defined in (Castro et al., 2009)<sup>1</sup>.</p> <p>In this game, some players have a left shoe and others a right shoe. Single shoes have a worth of zero while pairs have a worth of 1.</p> <p>The payoff of a coalition \\(S\\) is:</p> \\[{ v(S) = \\min( \\mid S \\cap L \\mid, \\mid S \\cap R \\mid ) }\\] <p>Where \\(L\\), respectively \\(R\\), is the set of players with left shoes, respectively right shoes.</p> PARAMETER  DESCRIPTION <code>left</code> <p>Number of players with a left shoe.</p> <p> TYPE: <code>int</code> </p> <code>right</code> <p>Number of players with a right shoe.</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, left: int, right: int) -&gt; None:\n    self.left = left\n    self.right = right\n    n_players = self.left + self.right\n    description = \"Dummy data for the shoe game in Castro et al. 2009\"\n    max_score = n_players // 2\n    super().__init__(n_players, score_range=(0, max_score), description=description)\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.AirportGame","title":"AirportGame","text":"<pre><code>AirportGame(n_players: int = 100)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>An airport game defined in (Castro et al., 2009)<sup>1</sup> Section 4.3</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> DEFAULT: <code>100</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int = 100) -&gt; None:\n    if n_players != 100:\n        raise ValueError(\n            f\"{self.__class__.__name__} only supports n_players=100 but got {n_players=}.\"\n        )\n    description = \"A dummy dataset for the airport game in Castro et al. 2009\"\n    super().__init__(n_players, score_range=(0, 100), description=description)\n    ranges = [\n        range(0, 8),\n        range(8, 20),\n        range(20, 26),\n        range(26, 40),\n        range(40, 48),\n        range(48, 57),\n        range(57, 70),\n        range(70, 80),\n        range(80, 90),\n        range(90, 100),\n    ]\n    exact = [\n        0.01,\n        0.020869565,\n        0.033369565,\n        0.046883079,\n        0.063549745,\n        0.082780515,\n        0.106036329,\n        0.139369662,\n        0.189369662,\n        0.289369662,\n    ]\n    c = list(range(1, 10))\n    score_table = np.zeros(100)\n    exact_values = np.zeros(100)\n\n    for r, v in zip(ranges, exact):\n        score_table[r] = c\n        exact_values[r] = v\n\n    self.exact_values = exact_values\n    self.score_table = score_table\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.MinimumSpanningTreeGame","title":"MinimumSpanningTreeGame","text":"<pre><code>MinimumSpanningTreeGame(n_players: int = 100)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>A minimum spanning tree game defined in (Castro et al., 2009)<sup>1</sup>.</p> <p>Let \\(G = (N \\cup \\{0\\},E)\\) be a valued graph where \\(N = \\{1,\\dots,100\\}\\), and the cost associated to an edge \\((i, j)\\) is:</p> \\[{ c_{ij} = \\left\\{\\begin{array}{lll} 1, &amp; \\text{ if} &amp; i = j + 1 \\text{ or } i = j - 1 \\\\ &amp; &amp; \\text{ or } (i = 1 \\text{ and } j = 100) \\text{ or } (i = 100 \\text{ and } j = 1) \\\\ 101, &amp; \\text{ if} &amp; i = 0 \\text{ or } j = 0 \\\\ \\infty, &amp; \\text{ otherwise} \\end{array}\\right. }\\] <p>A minimum spanning tree game \\((N, c)\\) is a cost game, where for a given coalition \\(S \\subset N\\), \\(v(S)\\) is the sum of the edge cost of the minimum spanning tree, i.e. \\(v(S)\\) = Minimum Spanning Tree of the graph \\(G|_{S\\cup\\{0\\}}\\), which is the partial graph restricted to the players \\(S\\) and the source node \\(0\\).</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of players that participate in the game.</p> <p> TYPE: <code>int</code> DEFAULT: <code>100</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int = 100) -&gt; None:\n    if n_players != 100:\n        raise ValueError(\n            f\"{self.__class__.__name__} only supports n_players=100 but got {n_players=}.\"\n        )\n    description = (\n        \"A dummy dataset for the minimum spanning tree game in Castro et al. 2009\"\n    )\n    super().__init__(n_players, score_range=(0, np.inf), description=description)\n\n    graph = np.zeros(shape=(self.n_players, self.n_players))\n\n    for i in range(self.n_players):\n        for j in range(self.n_players):\n            if (\n                i == j + 1\n                or i == j - 1\n                or (i == 1 and j == self.n_players - 1)\n                or (i == self.n_players - 1 and j == 1)\n            ):\n                graph[i, j] = 1\n            elif i == 0 or j == 0:\n                graph[i, j] = 0\n            else:\n                graph[i, j] = np.inf\n    assert np.all(graph == graph.T)\n\n    self.graph = graph\n</code></pre>"},{"location":"api/pydvl/value/games/#pydvl.value.games.MinerGame","title":"MinerGame","text":"<pre><code>MinerGame(n_players: int)\n</code></pre> <p>             Bases: <code>Game</code></p> <p>Toy game that is used for testing and demonstration purposes.</p> <p>Consider a group of n miners, who have discovered large bars of gold.</p> <p>If two miners can carry one piece of gold, then the payoff of a coalition \\(S\\) is:</p> \\[{ v(S) = \\left\\{\\begin{array}{lll} \\mid S \\mid / 2, &amp; \\text{ if} &amp; \\mid S \\mid \\text{ is even} \\\\ ( \\mid S \\mid - 1)/2, &amp; \\text{ otherwise} \\end{array}\\right. }\\] <p>If there are more than two miners and there is an even number of miners, then the core consists of the single payoff where each miner gets 1/2.</p> <p>If there is an odd number of miners, then the core is empty.</p> <p>Taken from Wikipedia</p> PARAMETER  DESCRIPTION <code>n_players</code> <p>Number of miners that participate in the game.</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/value/games.py</code> <pre><code>def __init__(self, n_players: int) -&gt; None:\n    if n_players &lt;= 2:\n        raise ValueError(f\"n_players, {n_players}, should be &gt; 2\")\n    description = \"Dummy data for Miner Game taken from https://en.wikipedia.org/wiki/Core_(game_theory)\"\n    super().__init__(\n        n_players,\n        score_range=(0, n_players // 2),\n        description=description,\n    )\n</code></pre>"},{"location":"api/pydvl/value/result/","title":"Result","text":""},{"location":"api/pydvl/value/result/#pydvl.value.result","title":"pydvl.value.result","text":"<p>This module collects types and methods for the inspection of the results of valuation algorithms.</p> <p>The most important class is ValuationResult, which provides access to raw values, as well as convenient behaviour as a <code>Sequence</code> with extended indexing and updating abilities, and conversion to pandas DataFrames.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result--operating-on-results","title":"Operating on results","text":"<p>Results can be added together with the standard <code>+</code> operator. Because values are typically running averages of iterative algorithms, addition behaves like a weighted average of the two results, with the weights being the number of updates in each result: adding two results is the same as generating one result with the mean of the values of the two results as values. The variances are updated accordingly. See ValuationResult for details.</p> <p>Results can also be sorted by value, variance or number of updates, see sort(). The arrays of ValuationResult.values, ValuationResult.variances, ValuationResult.counts, ValuationResult.indices, ValuationResult.names are sorted in the same way.</p> <p>Indexing and slicing of results is supported and ValueItem objects are returned. These objects can be compared with the usual operators, which take only the ValueItem.value into account.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result--creating-result-objects","title":"Creating result objects","text":"<p>The most commonly used factory method is ValuationResult.zeros(), which creates a result object with all values, variances and counts set to zero. ValuationResult.empty() creates an empty result object, which can be used as a starting point for adding results together. Empty results are discarded when added to other results. Finally, ValuationResult.from_random() samples random values uniformly.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValueItem","title":"ValueItem  <code>dataclass</code>","text":"<pre><code>ValueItem(\n    index: IndexT,\n    name: NameT,\n    value: float,\n    variance: Optional[float],\n    count: Optional[int],\n)\n</code></pre> <p>             Bases: <code>Generic[IndexT, NameT]</code></p> <p>The result of a value computation for one datum.</p> <p><code>ValueItems</code> can be compared with the usual operators, forming a total order. Comparisons take only the <code>value</code> into account.</p> <p>Todo</p> <p>Maybe have a mode of comparing similar to <code>np.isclose</code>, or taking the <code>variance</code> into account.</p> ATTRIBUTE DESCRIPTION <code>index</code> <p>Index of the sample with this value in the original Dataset</p> <p> TYPE: <code>IndexT</code> </p> <code>name</code> <p>Name of the sample if it was provided. Otherwise, <code>str(index)</code></p> <p> TYPE: <code>NameT</code> </p> <code>value</code> <p>The value</p> <p> TYPE: <code>float</code> </p> <code>variance</code> <p>Variance of the value if it was computed with an approximate method</p> <p> TYPE: <code>Optional[float]</code> </p> <code>count</code> <p>Number of updates for this value</p> <p> TYPE: <code>Optional[int]</code> </p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValueItem.stderr","title":"stderr  <code>property</code>","text":"<pre><code>stderr: Optional[float]\n</code></pre> <p>Standard error of the value.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult","title":"ValuationResult","text":"<pre><code>ValuationResult(\n    *,\n    values: NDArray[float_],\n    variances: Optional[NDArray[float_]] = None,\n    counts: Optional[NDArray[int_]] = None,\n    indices: Optional[NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    algorithm: str = \"\",\n    status: Status = Status.Pending,\n    sort: bool = False,\n    **extra_values\n)\n</code></pre> <p>             Bases: <code>Sequence</code>, <code>Iterable[ValueItem[IndexT, NameT]]</code>, <code>Generic[IndexT, NameT]</code></p> <p>Objects of this class hold the results of valuation algorithms.</p> <p>These include indices in the original Dataset, any data names (e.g. group names in GroupedDataset), the values themselves, and variance of the computation in the case of Monte Carlo methods. <code>ValuationResults</code> can be iterated over like any <code>Sequence</code>: <code>iter(valuation_result)</code> returns a generator of ValueItem in the order in which the object is sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--indexing","title":"Indexing","text":"<p>Indexing can be position-based, when accessing any of the attributes values, variances, counts and indices, as well as when iterating over the object, or using the item access operator, both getter and setter. The \"position\" is either the original sequence in which the data was passed to the constructor, or the sequence in which the object is sorted, see below.</p> <p>Alternatively, indexing can be data-based, i.e. using the indices in the original dataset. This is the case for the methods get() and update().</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--sorting","title":"Sorting","text":"<p>Results can be sorted in-place with sort(), or alternatively using python's standard <code>sorted()</code> and <code>reversed()</code> Note that sorting values affects how iterators and the object itself as <code>Sequence</code> behave: <code>values[0]</code> returns a ValueItem with the highest or lowest ranking point if this object is sorted by descending or ascending value, respectively. If unsorted, <code>values[0]</code> returns the <code>ValueItem</code> at position 0, which has data index <code>indices[0]</code> in the Dataset.</p> <p>The same applies to direct indexing of the <code>ValuationResult</code>: the index is positional, according to the sorting. It does not refer to the \"data index\". To sort according to data index, use sort() with <code>key=\"index\"</code>.</p> <p>In order to access ValueItem objects by their data index, use get().</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--operating-on-results","title":"Operating on results","text":"<p>Results can be added to each other with the <code>+</code> operator. Means and variances are correctly updated, using the <code>counts</code> attribute.</p> <p>Results can also be updated with new values using update(). Means and variances are updated accordingly using the Welford algorithm.</p> <p>Empty objects behave in a special way, see empty().</p> PARAMETER  DESCRIPTION <code>values</code> <p>An array of values. If omitted, defaults to an empty array or to an array of zeros if <code>indices</code> are given.</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>indices</code> <p>An optional array of indices in the original dataset. If omitted, defaults to <code>np.arange(len(values))</code>. Warning: It is common to pass the indices of a Dataset here. Attention must be paid in a parallel context to copy them to the local process. Just do <code>indices=np.copy(data.indices)</code>.</p> <p> TYPE: <code>Optional[NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> <code>variances</code> <p>An optional array of variances in the computation of each value.</p> <p> TYPE: <code>Optional[NDArray[float_]]</code> DEFAULT: <code>None</code> </p> <code>counts</code> <p>An optional array with the number of updates for each value. Defaults to an array of ones.</p> <p> TYPE: <code>Optional[NDArray[int_]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>Names for the data points. Defaults to index numbers if not set.</p> <p> TYPE: <code>Optional[Sequence[NameT] | NDArray[NameT]]</code> DEFAULT: <code>None</code> </p> <code>algorithm</code> <p>The method used.</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> <code>status</code> <p>The end status of the algorithm.</p> <p> TYPE: <code>Status</code> DEFAULT: <code>Pending</code> </p> <code>sort</code> <p>Whether to sort the indices by ascending value. See above how this affects usage as an iterable or sequence.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>extra_values</code> <p>Additional values that can be passed as keyword arguments. This can contain, for example, the least core value.</p> <p> DEFAULT: <code>{}</code> </p> RAISES DESCRIPTION <code>ValueError</code> <p>If input arrays have mismatching lengths.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __init__(\n    self,\n    *,\n    values: NDArray[np.float_],\n    variances: Optional[NDArray[np.float_]] = None,\n    counts: Optional[NDArray[np.int_]] = None,\n    indices: Optional[NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    algorithm: str = \"\",\n    status: Status = Status.Pending,\n    sort: bool = False,\n    **extra_values,\n):\n    if variances is not None and len(variances) != len(values):\n        raise ValueError(\"Lengths of values and variances do not match\")\n    if data_names is not None and len(data_names) != len(values):\n        raise ValueError(\"Lengths of values and data_names do not match\")\n    if indices is not None and len(indices) != len(values):\n        raise ValueError(\"Lengths of values and indices do not match\")\n\n    self._algorithm = algorithm\n    self._status = Status(status)  # Just in case we are given a string\n    self._values = values\n    self._variances = np.zeros_like(values) if variances is None else variances\n    self._counts = np.ones_like(values) if counts is None else counts\n    self._sort_order = None\n    self._extra_values = extra_values or {}\n\n    # Yuk...\n    if data_names is None:\n        if indices is not None:\n            self._names = np.copy(indices)\n        else:\n            self._names = np.arange(len(self._values), dtype=np.int_)\n    elif not isinstance(data_names, np.ndarray):\n        self._names = np.array(data_names)\n    else:\n        self._names = data_names.copy()\n    if len(np.unique(self._names)) != len(self._names):\n        raise ValueError(\"Data names must be unique\")\n\n    if indices is None:\n        indices = np.arange(len(self._values), dtype=np.int_)\n    self._indices = indices\n    self._positions = {idx: pos for pos, idx in enumerate(indices)}\n\n    self._sort_positions: NDArray[np.int_] = np.arange(\n        len(self._values), dtype=np.int_\n    )\n    if sort:\n        self.sort()\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.values","title":"values  <code>property</code>","text":"<pre><code>values: NDArray[float_]\n</code></pre> <p>The values, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.variances","title":"variances  <code>property</code>","text":"<pre><code>variances: NDArray[float_]\n</code></pre> <p>The variances, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.stderr","title":"stderr  <code>property</code>","text":"<pre><code>stderr: NDArray[float_]\n</code></pre> <p>The raw standard errors, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.counts","title":"counts  <code>property</code>","text":"<pre><code>counts: NDArray[int_]\n</code></pre> <p>The raw counts, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.indices","title":"indices  <code>property</code>","text":"<pre><code>indices: NDArray[IndexT]\n</code></pre> <p>The indices for the values, possibly sorted.</p> <p>If the object is unsorted, then these are the same as declared at construction or <code>np.arange(len(values))</code> if none were passed.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.names","title":"names  <code>property</code>","text":"<pre><code>names: NDArray[NameT]\n</code></pre> <p>The names for the values, possibly sorted. If the object is unsorted, then these are the same as declared at construction or <code>np.arange(len(values))</code> if none were passed.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.sort","title":"sort","text":"<pre><code>sort(\n    reverse: bool = False,\n    key: Literal[\"value\", \"variance\", \"index\", \"name\"] = \"value\",\n) -&gt; None\n</code></pre> <p>Sorts the indices in place by <code>key</code>.</p> <p>Once sorted, iteration over the results, and indexing of all the properties ValuationResult.values, ValuationResult.variances, ValuationResult.counts, ValuationResult.indices and ValuationResult.names will follow the same order.</p> PARAMETER  DESCRIPTION <code>reverse</code> <p>Whether to sort in descending order by value.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>key</code> <p>The key to sort by. Defaults to ValueItem.value.</p> <p> TYPE: <code>Literal['value', 'variance', 'index', 'name']</code> DEFAULT: <code>'value'</code> </p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def sort(\n    self,\n    reverse: bool = False,\n    # Need a \"Comparable\" type here\n    key: Literal[\"value\", \"variance\", \"index\", \"name\"] = \"value\",\n) -&gt; None:\n    \"\"\"Sorts the indices in place by `key`.\n\n    Once sorted, iteration over the results, and indexing of all the\n    properties\n    [ValuationResult.values][pydvl.value.result.ValuationResult.values],\n    [ValuationResult.variances][pydvl.value.result.ValuationResult.variances],\n    [ValuationResult.counts][pydvl.value.result.ValuationResult.counts],\n    [ValuationResult.indices][pydvl.value.result.ValuationResult.indices]\n    and [ValuationResult.names][pydvl.value.result.ValuationResult.names]\n    will follow the same order.\n\n    Args:\n        reverse: Whether to sort in descending order by value.\n        key: The key to sort by. Defaults to\n            [ValueItem.value][pydvl.value.result.ValueItem].\n    \"\"\"\n    keymap = {\n        \"index\": \"_indices\",\n        \"value\": \"_values\",\n        \"variance\": \"_variances\",\n        \"name\": \"_names\",\n    }\n    self._sort_positions = np.argsort(getattr(self, keymap[key]))\n    if reverse:\n        self._sort_positions = self._sort_positions[::-1]\n    self._sort_order = reverse\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__getattr__","title":"__getattr__","text":"<pre><code>__getattr__(attr: str) -&gt; Any\n</code></pre> <p>Allows access to extra values as if they were properties of the instance.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __getattr__(self, attr: str) -&gt; Any:\n    \"\"\"Allows access to extra values as if they were properties of the instance.\"\"\"\n    # This is here to avoid a RecursionError when copying or pickling the object\n    if attr == \"_extra_values\":\n        raise AttributeError()\n    try:\n        return self._extra_values[attr]\n    except KeyError as e:\n        raise AttributeError(\n            f\"{self.__class__.__name__} object has no attribute {attr}\"\n        ) from e\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__iter__","title":"__iter__","text":"<pre><code>__iter__() -&gt; Iterator[ValueItem[IndexT, NameT]]\n</code></pre> <p>Iterate over the results returning ValueItem objects. To sort in place before iteration, use sort().</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __iter__(self) -&gt; Iterator[ValueItem[IndexT, NameT]]:\n    \"\"\"Iterate over the results returning [ValueItem][pydvl.value.result.ValueItem] objects.\n    To sort in place before iteration, use [sort()][pydvl.value.result.ValuationResult.sort].\n    \"\"\"\n    for pos in self._sort_positions:\n        yield ValueItem(\n            self._indices[pos],\n            self._names[pos],\n            self._values[pos],\n            self._variances[pos],\n            self._counts[pos],\n        )\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__add__","title":"__add__","text":"<pre><code>__add__(\n    other: ValuationResult[IndexT, NameT]\n) -&gt; ValuationResult[IndexT, NameT]\n</code></pre> <p>Adds two ValuationResults.</p> <p>The values must have been computed with the same algorithm. An exception to this is if one argument has empty values, in which case the other argument is returned.</p> <p>Warning</p> <p>Abusing this will introduce numerical errors.</p> <p>Means and standard errors are correctly handled. Statuses are added with bit-wise <code>&amp;</code>, see Status. <code>data_names</code> are taken from the left summand, or if unavailable from the right one. The <code>algorithm</code> string is carried over if both terms have the same one or concatenated.</p> <p>It is possible to add ValuationResults of different lengths, and with different or overlapping indices. The result will have the union of indices, and the values.</p> <p>Warning</p> <p>FIXME: Arbitrary <code>extra_values</code> aren't handled.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __add__(\n    self, other: ValuationResult[IndexT, NameT]\n) -&gt; ValuationResult[IndexT, NameT]:\n    \"\"\"Adds two ValuationResults.\n\n    The values must have been computed with the same algorithm. An exception\n    to this is if one argument has empty values, in which case the other\n    argument is returned.\n\n    !!! Warning\n        Abusing this will introduce numerical errors.\n\n    Means and standard errors are correctly handled. Statuses are added with\n    bit-wise `&amp;`, see [Status][pydvl.value.result.Status].\n    `data_names` are taken from the left summand, or if unavailable from\n    the right one. The `algorithm` string is carried over if both terms\n    have the same one or concatenated.\n\n    It is possible to add ValuationResults of different lengths, and with\n    different or overlapping indices. The result will have the union of\n    indices, and the values.\n\n    !!! Warning\n        FIXME: Arbitrary `extra_values` aren't handled.\n\n    \"\"\"\n    # empty results\n    if len(self.values) == 0:\n        return other\n    if len(other.values) == 0:\n        return self\n\n    self._check_compatible(other)\n\n    indices = np.union1d(self._indices, other._indices).astype(self._indices.dtype)\n    this_pos = np.searchsorted(indices, self._indices)\n    other_pos = np.searchsorted(indices, other._indices)\n\n    n: NDArray[np.int_] = np.zeros_like(indices, dtype=int)\n    m: NDArray[np.int_] = np.zeros_like(indices, dtype=int)\n    xn: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\n    xm: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\n    vn: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\n    vm: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\n\n    n[this_pos] = self._counts\n    xn[this_pos] = self._values\n    vn[this_pos] = self._variances\n    m[other_pos] = other._counts\n    xm[other_pos] = other._values\n    vm[other_pos] = other._variances\n\n    # np.maximum(1, n + m) covers case n = m = 0.\n    n_m_sum = np.maximum(1, n + m)\n\n    # Sample mean of n+m samples from two means of n and m samples\n    xnm = (n * xn + m * xm) / n_m_sum\n\n    # Sample variance of n+m samples from two sample variances of n and m samples\n    vnm = (n * (vn + xn**2) + m * (vm + xm**2)) / n_m_sum - xnm**2\n\n    if np.any(vnm &lt; 0):\n        if np.any(vnm &lt; -1e-6):\n            logger.warning(\n                \"Numerical error in variance computation. \"\n                f\"Negative sample variances clipped to 0 in {vnm}\"\n            )\n        vnm[np.where(vnm &lt; 0)] = 0\n\n    # Merging of names:\n    # If an index has the same name in both results, it must be the same.\n    # If an index has a name in one result but not the other, the name is\n    # taken from the result with the name.\n    if self._names.dtype != other._names.dtype:\n        if np.can_cast(other._names.dtype, self._names.dtype, casting=\"safe\"):\n            other._names = other._names.astype(self._names.dtype)\n            logger.warning(\n                f\"Casting ValuationResult.names from {other._names.dtype} to {self._names.dtype}\"\n            )\n        else:\n            raise TypeError(\n                f\"Cannot cast ValuationResult.names from \"\n                f\"{other._names.dtype} to {self._names.dtype}\"\n            )\n\n    both_pos = np.intersect1d(this_pos, other_pos)\n\n    if len(both_pos) &gt; 0:\n        this_names: NDArray = np.empty_like(indices, dtype=object)\n        other_names: NDArray = np.empty_like(indices, dtype=object)\n        this_names[this_pos] = self._names\n        other_names[other_pos] = other._names\n\n        this_shared_names = np.take(this_names, both_pos)\n        other_shared_names = np.take(other_names, both_pos)\n\n        if np.any(this_shared_names != other_shared_names):\n            raise ValueError(f\"Mismatching names in ValuationResults\")\n\n    names = np.empty_like(indices, dtype=self._names.dtype)\n    names[this_pos] = self._names\n    names[other_pos] = other._names\n\n    return ValuationResult(\n        algorithm=self.algorithm or other.algorithm or \"\",\n        status=self.status &amp; other.status,\n        indices=indices,\n        values=xnm,\n        variances=vnm,\n        counts=n + m,\n        data_names=names,\n        # FIXME: What to do with extra_values? This is not commutative:\n        # extra_values=self._extra_values.update(other._extra_values),\n    )\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.update","title":"update","text":"<pre><code>update(idx: int, new_value: float) -&gt; ValuationResult[IndexT, NameT]\n</code></pre> <p>Updates the result in place with a new value, using running mean and variance.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Data index of the value to update.</p> <p> TYPE: <code>int</code> </p> <code>new_value</code> <p>New value to add to the result.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>ValuationResult[IndexT, NameT]</code> <p>A reference to the same, modified result.</p> RAISES DESCRIPTION <code>IndexError</code> <p>If the index is not found.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def update(self, idx: int, new_value: float) -&gt; ValuationResult[IndexT, NameT]:\n    \"\"\"Updates the result in place with a new value, using running mean\n    and variance.\n\n    Args:\n        idx: Data index of the value to update.\n        new_value: New value to add to the result.\n\n    Returns:\n        A reference to the same, modified result.\n\n    Raises:\n        IndexError: If the index is not found.\n    \"\"\"\n    try:\n        pos = self._positions[idx]\n    except KeyError:\n        raise IndexError(f\"Index {idx} not found in ValuationResult\")\n    val, var = running_moments(\n        self._values[pos], self._variances[pos], self._counts[pos], new_value\n    )\n    self[pos] = ValueItem(\n        index=cast(IndexT, idx),  # FIXME\n        name=self._names[pos],\n        value=val,\n        variance=var,\n        count=self._counts[pos] + 1,\n    )\n    return self\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.scale","title":"scale","text":"<pre><code>scale(factor: float, indices: Optional[NDArray[IndexT]] = None)\n</code></pre> <p>Scales the values and variances of the result by a coefficient.</p> PARAMETER  DESCRIPTION <code>factor</code> <p>Factor to scale by.</p> <p> TYPE: <code>float</code> </p> <code>indices</code> <p>Indices to scale. If None, all values are scaled.</p> <p> TYPE: <code>Optional[NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def scale(self, factor: float, indices: Optional[NDArray[IndexT]] = None):\n    \"\"\"\n    Scales the values and variances of the result by a coefficient.\n\n    Args:\n        factor: Factor to scale by.\n        indices: Indices to scale. If None, all values are scaled.\n    \"\"\"\n    self._values[self._sort_positions[indices]] *= factor\n    self._variances[self._sort_positions[indices]] *= factor**2\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.get","title":"get","text":"<pre><code>get(idx: Integral) -&gt; ValueItem\n</code></pre> <p>Retrieves a ValueItem by data index, as opposed to sort index, like the indexing operator.</p> RAISES DESCRIPTION <code>IndexError</code> <p>If the index is not found.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def get(self, idx: Integral) -&gt; ValueItem:\n    \"\"\"Retrieves a ValueItem by data index, as opposed to sort index, like\n    the indexing operator.\n\n    Raises:\n         IndexError: If the index is not found.\n    \"\"\"\n    try:\n        pos = self._positions[idx]\n    except KeyError:\n        raise IndexError(f\"Index {idx} not found in ValuationResult\")\n\n    return ValueItem(\n        self._indices[pos],\n        self._names[pos],\n        self._values[pos],\n        self._variances[pos],\n        self._counts[pos],\n    )\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.to_dataframe","title":"to_dataframe","text":"<pre><code>to_dataframe(\n    column: Optional[str] = None, use_names: bool = False\n) -&gt; DataFrame\n</code></pre> <p>Returns values as a dataframe.</p> PARAMETER  DESCRIPTION <code>column</code> <p>Name for the column holding the data value. Defaults to the name of the algorithm used.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>use_names</code> <p>Whether to use data names instead of indices for the DataFrame's index.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>DataFrame</code> <p>A dataframe with two columns, one for the values, with name given as explained in <code>column</code>, and another with standard errors for approximate algorithms. The latter will be named <code>column+'_stderr'</code>.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def to_dataframe(\n    self, column: Optional[str] = None, use_names: bool = False\n) -&gt; pd.DataFrame:\n    \"\"\"Returns values as a dataframe.\n\n    Args:\n        column: Name for the column holding the data value. Defaults to\n            the name of the algorithm used.\n        use_names: Whether to use data names instead of indices for the\n            DataFrame's index.\n\n    Returns:\n        A dataframe with two columns, one for the values, with name\n            given as explained in `column`, and another with standard errors for\n            approximate algorithms. The latter will be named `column+'_stderr'`.\n    \"\"\"\n    column = column or self._algorithm\n    df = pd.DataFrame(\n        self._values[self._sort_positions],\n        index=(\n            self._names[self._sort_positions]\n            if use_names\n            else self._indices[self._sort_positions]\n        ),\n        columns=[column],\n    )\n    df[column + \"_stderr\"] = self.stderr[self._sort_positions]\n    df[column + \"_updates\"] = self.counts[self._sort_positions]\n    return df\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.from_random","title":"from_random  <code>classmethod</code>","text":"<pre><code>from_random(\n    size: int,\n    total: Optional[float] = None,\n    seed: Optional[Seed] = None,\n    **kwargs\n) -&gt; \"ValuationResult\"\n</code></pre> <p>Creates a ValuationResult object and fills it with an array of random values from a uniform distribution in [-1,1]. The values can be made to sum up to a given total number (doing so will change their range).</p> PARAMETER  DESCRIPTION <code>size</code> <p>Number of values to generate</p> <p> TYPE: <code>int</code> </p> <code>total</code> <p>If set, the values are normalized to sum to this number (\"efficiency\" property of Shapley values).</p> <p> TYPE: <code>Optional[float]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional options to pass to the constructor of ValuationResult. Use to override status, names, etc.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>'ValuationResult'</code> <p>A valuation result with its status set to</p> <code>'ValuationResult'</code> <p>Status.Converged by default.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>size</code> is less than 1.</p> <p>Changed in version 0.6.0</p> <p>Added parameter <code>total</code>. Check for zero size</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\ndef from_random(\n    cls,\n    size: int,\n    total: Optional[float] = None,\n    seed: Optional[Seed] = None,\n    **kwargs,\n) -&gt; \"ValuationResult\":\n    \"\"\"Creates a [ValuationResult][pydvl.value.result.ValuationResult] object and fills it with an array\n    of random values from a uniform distribution in [-1,1]. The values can\n    be made to sum up to a given total number (doing so will change their range).\n\n    Args:\n        size: Number of values to generate\n        total: If set, the values are normalized to sum to this number\n            (\"efficiency\" property of Shapley values).\n        kwargs: Additional options to pass to the constructor of\n            [ValuationResult][pydvl.value.result.ValuationResult]. Use to override status, names, etc.\n\n    Returns:\n        A valuation result with its status set to\n        [Status.Converged][pydvl.utils.status.Status] by default.\n\n    Raises:\n         ValueError: If `size` is less than 1.\n\n    !!! tip \"Changed in version 0.6.0\"\n        Added parameter `total`. Check for zero size\n    \"\"\"\n    if size &lt; 1:\n        raise ValueError(\"Size must be a positive integer\")\n\n    rng = np.random.default_rng(seed)\n    values = rng.uniform(low=-1, high=1, size=size)\n    if total is not None:\n        values *= total / np.sum(values)\n\n    options = dict(values=values, status=Status.Converged, algorithm=\"random\")\n    options.update(kwargs)\n    return cls(**options)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.empty","title":"empty  <code>classmethod</code>","text":"<pre><code>empty(\n    algorithm: str = \"\",\n    indices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    n_samples: int = 0,\n) -&gt; ValuationResult\n</code></pre> <p>Creates an empty ValuationResult object.</p> <p>Empty results are characterised by having an empty array of values. When another result is added to an empty one, the empty one is discarded.</p> PARAMETER  DESCRIPTION <code>algorithm</code> <p>Name of the algorithm used to compute the values</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> <code>indices</code> <p>Optional sequence or array of indices.</p> <p> TYPE: <code>Optional[Sequence[IndexT] | NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>Optional sequences or array of names for the data points. Defaults to index numbers if not set.</p> <p> TYPE: <code>Optional[Sequence[NameT] | NDArray[NameT]]</code> DEFAULT: <code>None</code> </p> <code>n_samples</code> <p>Number of valuation result entries.</p> <p> TYPE: <code>int</code> DEFAULT: <code>0</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\ndef empty(\n    cls,\n    algorithm: str = \"\",\n    indices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    n_samples: int = 0,\n) -&gt; ValuationResult:\n    \"\"\"Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.\n\n    Empty results are characterised by having an empty array of values. When\n    another result is added to an empty one, the empty one is discarded.\n\n    Args:\n        algorithm: Name of the algorithm used to compute the values\n        indices: Optional sequence or array of indices.\n        data_names: Optional sequences or array of names for the data points.\n            Defaults to index numbers if not set.\n        n_samples: Number of valuation result entries.\n\n    Returns:\n        Object with the results.\n    \"\"\"\n    if indices is not None or data_names is not None or n_samples != 0:\n        return cls.zeros(\n            algorithm=algorithm,\n            indices=indices,\n            data_names=data_names,\n            n_samples=n_samples,\n        )\n    return cls(algorithm=algorithm, status=Status.Pending, values=np.array([]))\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.zeros","title":"zeros  <code>classmethod</code>","text":"<pre><code>zeros(\n    algorithm: str = \"\",\n    indices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    n_samples: int = 0,\n) -&gt; ValuationResult\n</code></pre> <p>Creates an empty ValuationResult object.</p> <p>Empty results are characterised by having an empty array of values. When another result is added to an empty one, the empty one is ignored.</p> PARAMETER  DESCRIPTION <code>algorithm</code> <p>Name of the algorithm used to compute the values</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> <code>indices</code> <p>Data indices to use. A copy will be made. If not given, the indices will be set to the range <code>[0, n_samples)</code>.</p> <p> TYPE: <code>Optional[Sequence[IndexT] | NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>Data names to use. A copy will be made. If not given, the names will be set to the string representation of the indices.</p> <p> TYPE: <code>Optional[Sequence[NameT] | NDArray[NameT]]</code> DEFAULT: <code>None</code> </p> <code>n_samples</code> <p>Number of data points whose values are computed. If not given, the length of <code>indices</code> will be used.</p> <p> TYPE: <code>int</code> DEFAULT: <code>0</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\ndef zeros(\n    cls,\n    algorithm: str = \"\",\n    indices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\n    data_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\n    n_samples: int = 0,\n) -&gt; ValuationResult:\n    \"\"\"Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.\n\n    Empty results are characterised by having an empty array of values. When\n    another result is added to an empty one, the empty one is ignored.\n\n    Args:\n        algorithm: Name of the algorithm used to compute the values\n        indices: Data indices to use. A copy will be made. If not given,\n            the indices will be set to the range `[0, n_samples)`.\n        data_names: Data names to use. A copy will be made. If not given,\n            the names will be set to the string representation of the indices.\n        n_samples: Number of data points whose values are computed. If\n            not given, the length of `indices` will be used.\n\n    Returns:\n        Object with the results.\n    \"\"\"\n    if indices is None:\n        indices = np.arange(n_samples, dtype=np.int_)\n    else:\n        indices = np.array(indices, dtype=np.int_)\n\n    if data_names is None:\n        data_names = np.array(indices)\n    else:\n        data_names = np.array(data_names)\n\n    return cls(\n        algorithm=algorithm,\n        status=Status.Pending,\n        indices=indices,\n        data_names=data_names,\n        values=np.zeros(len(indices)),\n        variances=np.zeros(len(indices)),\n        counts=np.zeros(len(indices), dtype=np.int_),\n    )\n</code></pre>"},{"location":"api/pydvl/value/sampler/","title":"Sampler","text":""},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler","title":"pydvl.value.sampler","text":"<p>Samplers iterate over subsets of indices.</p> <p>The classes in this module are used to iterate over indices and subsets of their complement in the whole set, as required for the computation of marginal utility for semi-values. The elements returned when iterating over any subclass of PowersetSampler are tuples of the form <code>(idx, subset)</code>, where <code>idx</code> is the index of the element being added to the subset, and <code>subset</code> is the subset of the complement of <code>idx</code>. The classes in this module are used to iterate over an index set \\(I\\) as required for the computation of marginal utility for semi-values. The elements returned when iterating over any subclass of :class:<code>PowersetSampler</code> are tuples of the form \\((i, S)\\), where \\(i\\) is an index of interest, and \\(S \\subset I \\setminus \\{i\\}\\) is a subset of the complement of \\(i\\).</p> <p>The iteration happens in two nested loops. An outer loop iterates over \\(I\\), and an inner loop iterates over the powerset of \\(I \\setminus \\{i\\}\\). The outer iteration can be either sequential or at random.</p> <p>Note</p> <p>This is the natural mode of iteration for the combinatorial definition of semi-values, in particular Shapley value. For the computation using permutations, adhering to this interface is not ideal, but we stick to it for consistency.</p> <p>The samplers are used in the semivalues module to compute any semi-value, in particular Shapley and Beta values, and Banzhaf indices.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler--slicing-of-samplers","title":"Slicing of samplers","text":"<p>The samplers can be sliced for parallel computation. For those which are embarrassingly parallel, this is done by slicing the set of \"outer\" indices and returning new samplers over those slices. This includes all truly powerset-based samplers, such as DeterministicUniformSampler and UniformSampler. In contrast, slicing a PermutationSampler creates a new sampler which iterates over the same indices.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler--references","title":"References","text":"<ol> <li> <p>Mitchell, Rory, Joshua Cooper, Eibe   Frank, and Geoffrey Holmes. Sampling Permutations for Shapley Value   Estimation. Journal of Machine   Learning Research 23, no. 43 (2022): 1\u201346.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T. and Jia, R., 2023. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning. In: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics, pp. 6388-6421.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler","title":"PowersetSampler","text":"<pre><code>PowersetSampler(\n    indices: NDArray[IndexT],\n    index_iteration: IndexIteration = IndexIteration.Sequential,\n    outer_indices: NDArray[IndexT] | None = None,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>ABC</code>, <code>Iterable[SampleT]</code>, <code>Generic[IndexT]</code></p> <p>Samplers are custom iterables over subsets of indices.</p> <p>Calling <code>iter()</code> on a sampler returns an iterator over tuples of the form \\((i, S)\\), where \\(i\\) is an index of interest, and \\(S \\subset I \\setminus \\{i\\}\\) is a subset of the complement of \\(i\\).</p> <p>This is done in two nested loops, where the outer loop iterates over the set of indices, and the inner loop iterates over subsets of the complement of the current index. The outer iteration can be either sequential or at random.</p> <p>Note</p> <p>Samplers are not iterators themselves, so that each call to <code>iter()</code> e.g. in a for loop creates a new iterator.</p> Example <pre><code>&gt;&gt;&gt;for idx, s in DeterministicUniformSampler(np.arange(2)):\n&gt;&gt;&gt;    print(s, end=\"\")\n[][2,][][1,]\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler--methods-required-in-subclasses","title":"Methods required in subclasses","text":"<p>Samplers must implement a weight() function to be used as a multiplier in Monte Carlo sums, so that the limit expectation coincides with the semi-value.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler--slicing-of-samplers","title":"Slicing of samplers","text":"<p>The samplers can be sliced for parallel computation. For those which are embarrassingly parallel, this is done by slicing the set of \"outer\" indices and returning new samplers over those slices.</p> <pre><code>index_iteration: the order in which indices are iterated over\nouter_indices: The set of items (indices) over which to iterate\n    when sampling. Subsets are taken from the complement of each index\n    in succession. For embarrassingly parallel computations, this set\n    is sliced and the samplers are used to iterate over the slices.\n</code></pre> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(\n    self,\n    indices: NDArray[IndexT],\n    index_iteration: IndexIteration = IndexIteration.Sequential,\n    outer_indices: NDArray[IndexT] | None = None,\n    **kwargs,\n):\n    \"\"\"\n    Args:\n        indices: The set of items (indices) to sample from.\n        index_iteration: the order in which indices are iterated over\n        outer_indices: The set of items (indices) over which to iterate\n            when sampling. Subsets are taken from the complement of each index\n            in succession. For embarrassingly parallel computations, this set\n            is sliced and the samplers are used to iterate over the slices.\n    \"\"\"\n    self._indices = indices\n    self._index_iteration = index_iteration\n    self._outer_indices = outer_indices if outer_indices is not None else indices\n    self._n = len(indices)\n    self._n_samples = 0\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.weight","title":"weight  <code>abstractmethod</code> <code>classmethod</code>","text":"<pre><code>weight(n: int, subset_len: int) -&gt; float\n</code></pre> <p>Factor by which to multiply Monte Carlo samples, so that the mean converges to the desired expression.</p> <p>By the Law of Large Numbers, the sample mean of \\(\\delta_i(S_j)\\) converges to the expectation under the distribution from which \\(S_j\\) is sampled.</p> \\[ \\frac{1}{m}  \\sum_{j = 1}^m \\delta_i (S_j) c (S_j) \\longrightarrow    \\underset{S \\sim \\mathcal{D}_{- i}}{\\mathbb{E}} [\\delta_i (S) c (    S)]\\] <p>We add a factor \\(c(S_j)\\) in order to have this expectation coincide with the desired expression.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>@classmethod\n@abc.abstractmethod\ndef weight(cls, n: int, subset_len: int) -&gt; float:\n    r\"\"\"Factor by which to multiply Monte Carlo samples, so that the\n    mean converges to the desired expression.\n\n    By the Law of Large Numbers, the sample mean of $\\delta_i(S_j)$\n    converges to the expectation under the distribution from which $S_j$ is\n    sampled.\n\n    $$ \\frac{1}{m}  \\sum_{j = 1}^m \\delta_i (S_j) c (S_j) \\longrightarrow\n       \\underset{S \\sim \\mathcal{D}_{- i}}{\\mathbb{E}} [\\delta_i (S) c (\n       S)]$$\n\n    We add a factor $c(S_j)$ in order to have this expectation coincide with\n    the desired expression.\n    \"\"\"\n    ...\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.StochasticSamplerMixin","title":"StochasticSamplerMixin","text":"<pre><code>StochasticSamplerMixin(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>Mixin class for samplers which use a random number generator.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicUniformSampler","title":"DeterministicUniformSampler","text":"<pre><code>DeterministicUniformSampler(indices: NDArray[IndexT], *args, **kwargs)\n</code></pre> <p>             Bases: <code>PowersetSampler[IndexT]</code></p> <p>For every index \\(i\\), each subset of the complement <code>indices - {i}</code> is returned.</p> <p>Note</p> <p>Indices are always iterated over sequentially, irrespective of the value of <code>index_iteration</code> upon construction.</p> Example <pre><code>&gt;&gt;&gt; for idx, s in DeterministicUniformSampler(np.arange(2)):\n&gt;&gt;&gt;    print(f\"{idx} - {s}\", end=\", \")\n1 - [], 1 - [2], 2 - [], 2 - [1],\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>The set of items (indices) to sample from.</p> <p> TYPE: <code>NDArray[IndexT]</code> </p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, indices: NDArray[IndexT], *args, **kwargs):\n    \"\"\"An iterator to perform uniform deterministic sampling of subsets.\n\n    For every index $i$, each subset of the complement `indices - {i}` is\n    returned.\n\n    !!! Note\n        Indices are always iterated over sequentially, irrespective of\n        the value of `index_iteration` upon construction.\n\n    ??? Example\n        ``` pycon\n        &gt;&gt;&gt; for idx, s in DeterministicUniformSampler(np.arange(2)):\n        &gt;&gt;&gt;    print(f\"{idx} - {s}\", end=\", \")\n        1 - [], 1 - [2], 2 - [], 2 - [1],\n        ```\n\n    Args:\n        indices: The set of items (indices) to sample from.\n    \"\"\"\n    # Force sequential iteration\n    kwargs.update({\"index_iteration\": PowersetSampler.IndexIteration.Sequential})\n    super().__init__(indices, *args, **kwargs)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicUniformSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicUniformSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler","title":"UniformSampler","text":"<pre><code>UniformSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>An iterator to perform uniform random sampling of subsets.</p> <p>Iterating over every index \\(i\\), either in sequence or at random depending on the value of <code>index_iteration</code>, one subset of the complement <code>indices - {i}</code> is sampled with equal probability \\(2^{n-1}\\). The iterator never ends.</p> Example <p>The code <pre><code>for idx, s in UniformSampler(np.arange(3)):\n   print(f\"{idx} - {s}\", end=\", \")\n</code></pre> Produces the output: <pre><code>0 - [1 4], 1 - [2 3], 2 - [0 1 3], 3 - [], 4 - [2], 0 - [1 3 4], 1 - [0 2]\n(...)\n</code></pre></p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler.weight","title":"weight  <code>classmethod</code>","text":"<pre><code>weight(n: int, subset_len: int) -&gt; float\n</code></pre> <p>Correction coming from Monte Carlo integration so that the mean of the marginals converges to the value: the uniform distribution over the powerset of a set with n-1 elements has mass 2^{n-1} over each subset.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>@classmethod\ndef weight(cls, n: int, subset_len: int) -&gt; float:\n    \"\"\"Correction coming from Monte Carlo integration so that the mean of\n    the marginals converges to the value: the uniform distribution over the\n    powerset of a set with n-1 elements has mass 2^{n-1} over each subset.\"\"\"\n    return float(2 ** (n - 1)) if n &gt; 0 else 1.0\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.MSRSampler","title":"MSRSampler","text":"<pre><code>MSRSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>An iterator to perform sampling of random subsets.</p> <p>This sampler does not return any index, it only returns subsets of the data. This sampler is used in (Wang et. al.)<sup>2</sup>.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.MSRSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.MSRSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticSampler","title":"AntitheticSampler","text":"<pre><code>AntitheticSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>An iterator to perform uniform random sampling of subsets, and their complements.</p> <p>Works as UniformSampler, but for every tuple \\((i,S)\\), it subsequently returns \\((i,S^c)\\), where \\(S^c\\) is the complement of the set \\(S\\) in the set of indices, excluding \\(i\\).</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler","title":"PermutationSampler","text":"<pre><code>PermutationSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>Sample permutations of indices and iterate through each returning increasing subsets, as required for the permutation definition of semi-values.</p> <p>This sampler does not implement the two loops described in PowersetSampler. Instead, for a permutation <code>(3,1,4,2)</code>, it returns in sequence the tuples of index and sets:  <code>(3, {})</code>, <code>(1, {3})</code>, <code>(4, {3,1})</code> and <code>(2, {3,1,4})</code>.</p> <p>Note that the full index set is never returned.</p> <p>Warning</p> <p>This sampler requires caching to be enabled or computation will be doubled wrt. a \"direct\" implementation of permutation MC</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler.__getitem__","title":"__getitem__","text":"<pre><code>__getitem__(key: slice | list[int]) -&gt; PowersetSampler[IndexT]\n</code></pre> <p>Permutation samplers cannot be split across indices, so we return a copy of the full sampler.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __getitem__(self, key: slice | list[int]) -&gt; PowersetSampler[IndexT]:\n    \"\"\"Permutation samplers cannot be split across indices, so we return\n    a copy of the full sampler.\"\"\"\n    return super().__getitem__(slice(None))\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticPermutationSampler","title":"AntitheticPermutationSampler","text":"<pre><code>AntitheticPermutationSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>PermutationSampler[IndexT]</code></p> <p>Samples permutations like PermutationSampler, but after each permutation, it returns the same permutation in reverse order.</p> <p>This sampler was suggested in (Mitchell et al. 2022)<sup>1</sup></p> <p>New in version 0.7.1</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticPermutationSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticPermutationSampler.__getitem__","title":"__getitem__","text":"<pre><code>__getitem__(key: slice | list[int]) -&gt; PowersetSampler[IndexT]\n</code></pre> <p>Permutation samplers cannot be split across indices, so we return a copy of the full sampler.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __getitem__(self, key: slice | list[int]) -&gt; PowersetSampler[IndexT]:\n    \"\"\"Permutation samplers cannot be split across indices, so we return\n    a copy of the full sampler.\"\"\"\n    return super().__getitem__(slice(None))\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticPermutationSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicPermutationSampler","title":"DeterministicPermutationSampler","text":"<pre><code>DeterministicPermutationSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>PermutationSampler[IndexT]</code></p> <p>Samples all n! permutations of the indices deterministically, and iterates through them, returning sets as required for the permutation-based definition of semi-values.</p> <p>Warning</p> <p>This sampler requires caching to be enabled or computation will be doubled wrt. a \"direct\" implementation of permutation MC</p> <p>Warning</p> <p>This sampler is not parallelizable, as it always iterates over the whole set of permutations in the same order. Different processes would always return the same values for all indices.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicPermutationSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicPermutationSampler.__getitem__","title":"__getitem__","text":"<pre><code>__getitem__(key: slice | list[int]) -&gt; PowersetSampler[IndexT]\n</code></pre> <p>Permutation samplers cannot be split across indices, so we return a copy of the full sampler.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __getitem__(self, key: slice | list[int]) -&gt; PowersetSampler[IndexT]:\n    \"\"\"Permutation samplers cannot be split across indices, so we return\n    a copy of the full sampler.\"\"\"\n    return super().__getitem__(slice(None))\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicPermutationSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.RandomHierarchicalSampler","title":"RandomHierarchicalSampler","text":"<pre><code>RandomHierarchicalSampler(*args, seed: Optional[Seed] = None, **kwargs)\n</code></pre> <p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>For every index, sample a set size, then a set of that size.</p> <p>Todo</p> <p>This is unnecessary, but a step towards proper stratified sampling.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\n    super().__init__(*args, **kwargs)\n    self._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.RandomHierarchicalSampler.iterindices","title":"iterindices","text":"<pre><code>iterindices() -&gt; Iterator[IndexT]\n</code></pre> <p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n    \"\"\"Iterates over indices in the order specified at construction.\n\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\n    if self._index_iteration is PowersetSampler.IndexIteration.Sequential:\n        for idx in self._outer_indices:\n            yield idx\n    elif self._index_iteration is PowersetSampler.IndexIteration.Random:\n        while True:\n            yield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.RandomHierarchicalSampler.__len__","title":"__len__","text":"<pre><code>__len__() -&gt; int\n</code></pre> <p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n    \"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\n    return len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/","title":"Semivalues","text":""},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues","title":"pydvl.value.semivalues","text":"<p>This module provides the core functionality for the computation of generic semi-values. A semi-value is any valuation function with the form:</p> \\[v_\\text{semi}(i) = \\sum_{i=1}^n w(k) \\sum_{S \\subset D_{-i}^{(k)}} [U(S_{+i})-U(S)],\\] <p>where the coefficients \\(w(k)\\) satisfy the property:</p> \\[\\sum_{k=1}^n w(k) = 1.\\] Note <p>For implementation consistency, we slightly depart from the common definition of semi-values, which includes a factor \\(1/n\\) in the sum over subsets. Instead, we subsume this factor into the coefficient \\(w(k)\\).</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues--main-components","title":"Main components","text":"<p>The computation of a semi-value requires two components:</p> <ol> <li>A subset sampler that generates subsets of the set \\(D\\) of interest.</li> <li>A coefficient \\(w(k)\\) that assigns a weight to each subset size \\(k\\).</li> </ol> <p>Samplers can be found in sampler, and can be classified into two categories: powerset samplers and permutation samplers. Powerset samplers generate subsets of \\(D_{-i}\\), while the permutation sampler generates permutations of \\(D\\). The former conform to the above definition of semi-values, while the latter reformulates it as:</p> \\[ v(i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} \\tilde{w}( | \\sigma_{:i} | )[U(\\sigma_{:i} \\cup \\{i\\}) \u2212 U(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears (see Data valuation for details), and</p> \\[ \\tilde{w} (k) = n \\binom{n - 1}{k} w (k) \\] <p>is the weight correction due to the reformulation.</p> <p>Warning</p> <p>Both PermutationSampler and DeterministicPermutationSampler require caching to be enabled or computation will be doubled wrt. a 'direct' implementation of permutation MC.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues--computing-semi-values","title":"Computing semi-values","text":"<p>Samplers and coefficients can be arbitrarily mixed by means of the main entry point of this module, compute_generic_semivalues. There are several pre-defined coefficients, including the Shapley value of (Ghorbani and Zou, 2019)<sup>1</sup>, the Banzhaf index of (Wang and Jia)<sup>3</sup>, and the Beta coefficient of (Kwon and Zou, 2022)<sup>2</sup>. For each of these methods, there is a convenience wrapper function. Respectively, these are: compute_shapley_semivalues, compute_banzhaf_semivalues, and compute_beta_shapley_semivalues. instead.</p> <p>Parallelization and batching</p> <p>In order to ensure reproducibility and fine-grained control of parallelization, samples are generated in the main process and then distributed to worker processes for evaluation. For small sample sizes, this can lead to a significant overhead. To avoid this, we temporarily provide an additional argument <code>batch_size</code> to all methods which can improve performance with small models up to an order of magnitude. Note that this argument will be removed before version 1.0 in favour of a more general solution.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y. and Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning. In: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022, Vol. 151. PMLR, Valencia, Spain.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T. and Jia, R., 2023. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning. In: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics, pp. 6388-6421.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SVCoefficient","title":"SVCoefficient","text":"<p>             Bases: <code>Protocol</code></p> <p>The protocol that coefficients for the computation of semi-values must fulfill.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SVCoefficient.__call__","title":"__call__","text":"<pre><code>__call__(n: int, k: int) -&gt; float\n</code></pre> <p>Computes the coefficient for a given subset size.</p> PARAMETER  DESCRIPTION <code>n</code> <p>Total number of elements in the set.</p> <p> TYPE: <code>int</code> </p> <code>k</code> <p>Size of the subset for which the coefficient is being computed</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __call__(self, n: int, k: int) -&gt; float:\n    \"\"\"Computes the coefficient for a given subset size.\n\n    Args:\n        n: Total number of elements in the set.\n        k: Size of the subset for which the coefficient is being computed\n    \"\"\"\n    ...\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.DefaultMarginal","title":"DefaultMarginal","text":"<p>             Bases: <code>MarginalFunction</code></p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.DefaultMarginal.__call__","title":"__call__","text":"<pre><code>__call__(\n    u: Utility, coefficient: SVCoefficient, samples: Iterable[SampleT]\n) -&gt; Tuple[MarginalT, ...]\n</code></pre> <p>Computation of marginal utility. This is a helper function for compute_generic_semivalues.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>coefficient</code> <p>The semivalue coefficient and sampler weight</p> <p> TYPE: <code>SVCoefficient</code> </p> <code>samples</code> <p>A collection of samples. Each sample is a tuple of index and subset of indices to compute a marginal utility.</p> <p> TYPE: <code>Iterable[SampleT]</code> </p> RETURNS DESCRIPTION <code>MarginalT</code> <p>A collection of marginals. Each marginal is a tuple with index and its marginal</p> <code>...</code> <p>utility.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __call__(\n    self, u: Utility, coefficient: SVCoefficient, samples: Iterable[SampleT]\n) -&gt; Tuple[MarginalT, ...]:\n    \"\"\"Computation of marginal utility. This is a helper function for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        coefficient: The semivalue coefficient and sampler weight\n        samples: A collection of samples. Each sample is a tuple of index and subset of\n            indices to compute a marginal utility.\n\n    Returns:\n        A collection of marginals. Each marginal is a tuple with index and its marginal\n        utility.\n    \"\"\"\n    n = len(u.data)\n    marginals: List[MarginalT] = []\n    for idx, s in samples:\n        marginal = (u({idx}.union(s)) - u(s)) * coefficient(n, len(s))\n        marginals.append((idx, marginal))\n    return tuple(marginals)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.RawUtility","title":"RawUtility","text":"<p>             Bases: <code>MarginalFunction</code></p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.RawUtility.__call__","title":"__call__","text":"<pre><code>__call__(\n    u: Utility, coefficient: SVCoefficient, samples: Iterable[SampleT]\n) -&gt; Tuple[MarginalT, ...]\n</code></pre> <p>Computation of raw utility without marginalization. This is a helper function for compute_generic_semivalues.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>coefficient</code> <p>The semivalue coefficient and sampler weight</p> <p> TYPE: <code>SVCoefficient</code> </p> <code>samples</code> <p>A collection of samples. Each sample is a tuple of index and subset of indices to compute a marginal utility.</p> <p> TYPE: <code>Iterable[SampleT]</code> </p> RETURNS DESCRIPTION <code>Tuple[MarginalT, ...]</code> <p>A collection of marginals. Each marginal is a tuple with index and its raw utility.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __call__(\n    self, u: Utility, coefficient: SVCoefficient, samples: Iterable[SampleT]\n) -&gt; Tuple[MarginalT, ...]:\n    \"\"\"Computation of raw utility without marginalization. This is a helper function for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        coefficient: The semivalue coefficient and sampler weight\n        samples: A collection of samples. Each sample is a tuple of index and subset of\n            indices to compute a marginal utility.\n\n    Returns:\n        A collection of marginals. Each marginal is a tuple with index and its raw utility.\n    \"\"\"\n    marginals: List[MarginalT] = []\n    for idx, s in samples:\n        marginals.append((s, u(s)))\n    return tuple(marginals)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.FutureProcessor","title":"FutureProcessor","text":"<p>The FutureProcessor class used to process the results of the parallel marginal evaluations.</p> <p>The marginals are evaluated in parallel by <code>n_jobs</code> threads, but some algorithms require a central method to postprocess the marginal results. This can be achieved through the future processor. This base class does not perform any postprocessing, it is a noop used in most data valuation algorithms.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.MSRFutureProcessor","title":"MSRFutureProcessor","text":"<pre><code>MSRFutureProcessor(u: Utility)\n</code></pre> <p>             Bases: <code>FutureProcessor</code></p> <p>This FutureProcessor processes the raw marginals in a way that MSR sampling requires.</p> <p>MSR sampling evaluates the utility once, and then updates all data semivalues based on this one evaluation. In order to do this, the RawUtility value needs to be postprocessed through this class. For more details on MSR, please refer to the paper (Wang et. al.)<sup>3</sup>. This processor keeps track of the current values and computes marginals for all data points, so that the values in the ValuationResult can be updated properly down the line.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __init__(self, u: Utility):\n    self.n = len(u.data)\n    self.all_indices = u.data.indices.copy()\n    self.point_in_subset = np.zeros((self.n,))\n    self.positive_sums = np.zeros((self.n,))\n    self.negative_sums = np.zeros((self.n,))\n    self.total_evaluations = 0\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.MSRFutureProcessor.__call__","title":"__call__","text":"<pre><code>__call__(\n    future_result: List[Tuple[List[IndexT], float]]\n) -&gt; List[List[MarginalT]]\n</code></pre> <p>Computation of marginal utility using Maximum Sample Reuse.</p> <pre><code>    This processor requires the Marginal Function to be set to RawUtility.\n    Then, this processor computes marginals based on the utility value and the index set provided.\n\n    The final formula that gives the Banzhaf semivalue using MSR is:\n    $$\\hat{\\phi}_{MSR}(i) = \frac{1}{|\\mathbf{S}_{\n</code></pre> <p>i i}|} \\sum_{S \\in \\mathbf{S}{ i i}} U(S)         - \frac{1}{|\\mathbf{S}{ ot{ i} i}|} \\sum_{S \\in \\mathbf{S}_{ ot{ i} i}} U(S)$$</p> <pre><code>    Args:\n        future_result: Result of the parallel computing jobs comprised of\n            a list of indices that were used to evaluate the utility, and the evaluation result (metric).\n\n    Returns:\n        A collection of marginals. Each marginal is a tuple with index and its marginal\n        utility.\n</code></pre> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __call__(\n    self, future_result: List[Tuple[List[IndexT], float]]\n) -&gt; List[List[MarginalT]]:\n    \"\"\"Computation of marginal utility using Maximum Sample Reuse.\n\n    This processor requires the Marginal Function to be set to RawUtility.\n    Then, this processor computes marginals based on the utility value and the index set provided.\n\n    The final formula that gives the Banzhaf semivalue using MSR is:\n    $$\\hat{\\phi}_{MSR}(i) = \\frac{1}{|\\mathbf{S}_{\\ni i}|} \\sum_{S \\in \\mathbf{S}_{\\ni i}} U(S)\n    - \\frac{1}{|\\mathbf{S}_{\\not{\\ni} i}|} \\sum_{S \\in \\mathbf{S}_{\\not{\\ni} i}} U(S)$$\n\n    Args:\n        future_result: Result of the parallel computing jobs comprised of\n            a list of indices that were used to evaluate the utility, and the evaluation result (metric).\n\n    Returns:\n        A collection of marginals. Each marginal is a tuple with index and its marginal\n        utility.\n    \"\"\"\n    marginals: List[List[MarginalT]] = []\n    for batch_id, (s, evaluation) in enumerate(future_result):\n        previous_values = self.compute_values()\n        self.total_evaluations += 1\n        self.point_in_subset[s] += 1\n        self.positive_sums[s] += evaluation\n        not_s = np.setdiff1d(self.all_indices, s)\n        self.negative_sums[not_s] += evaluation\n        new_values = self.compute_values()\n        # Hack to work around the update mechanic that does not work out of the box for MSR\n        marginal_vals = (\n            self.total_evaluations * new_values\n            - (self.total_evaluations - 1) * previous_values\n        )\n        marginals.append([])\n        for data_index in range(self.n):\n            marginals[batch_id].append(\n                (data_index, float(marginal_vals[data_index]))\n            )\n    return marginals\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SemiValueMode","title":"SemiValueMode","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Enumeration of semi-value modes.</p> <p>Deprecation notice</p> <p>This enum and the associated methods are deprecated and will be removed in 0.8.0.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_generic_semivalues","title":"compute_generic_semivalues","text":"<pre><code>compute_generic_semivalues(\n    sampler: PowersetSampler[IndexT],\n    u: Utility,\n    coefficient: SVCoefficient,\n    done: StoppingCriterion,\n    *,\n    marginal: MarginalFunction = DefaultMarginal(),\n    future_processor: FutureProcessor = FutureProcessor(),\n    batch_size: int = 1,\n    skip_converged: bool = False,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False\n) -&gt; ValuationResult\n</code></pre> <p>Computes semi-values for a given utility function and subset sampler.</p> PARAMETER  DESCRIPTION <code>sampler</code> <p>The subset sampler to use for utility computations.</p> <p> TYPE: <code>PowersetSampler[IndexT]</code> </p> <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>coefficient</code> <p>The semi-value coefficient</p> <p> TYPE: <code>SVCoefficient</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>marginal</code> <p>Marginal function to be used for computing the semivalues</p> <p> TYPE: <code>MarginalFunction</code> DEFAULT: <code>DefaultMarginal()</code> </p> <code>future_processor</code> <p>Additional postprocessing steps required for some algorithms</p> <p> TYPE: <code>FutureProcessor</code> DEFAULT: <code>FutureProcessor()</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>skip_converged</code> <p>Whether to skip marginal evaluations for indices that have already converged. CAUTION: This is only entirely safe if the stopping criterion is MaxUpdates. For any other stopping criterion, the convergence status of indices may change during the computation, or they may be marked as having converged even though in fact the estimated values are far from the true values (e.g. for AbsoluteStandardError, you will probably have to carefully adjust the threshold).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_generic_semivalues(\n    sampler: PowersetSampler[IndexT],\n    u: Utility,\n    coefficient: SVCoefficient,\n    done: StoppingCriterion,\n    *,\n    marginal: MarginalFunction = DefaultMarginal(),\n    future_processor: FutureProcessor = FutureProcessor(),\n    batch_size: int = 1,\n    skip_converged: bool = False,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n) -&gt; ValuationResult:\n    \"\"\"Computes semi-values for a given utility function and subset sampler.\n\n    Args:\n        sampler: The subset sampler to use for utility computations.\n        u: Utility object with model, data, and scoring function.\n        coefficient: The semi-value coefficient\n        done: Stopping criterion.\n        marginal: Marginal function to be used for computing the semivalues\n        future_processor: Additional postprocessing steps required for some algorithms\n        batch_size: Number of marginal evaluations per single parallel job.\n        skip_converged: Whether to skip marginal evaluations for indices that\n            have already converged. **CAUTION**: This is only entirely safe if\n            the stopping criterion is [MaxUpdates][pydvl.value.stopping.MaxUpdates].\n            For any other stopping criterion, the convergence status of indices\n            may change during the computation, or they may be marked as having\n            converged even though in fact the estimated values are far from the\n            true values (e.g. for\n            [AbsoluteStandardError][pydvl.value.stopping.AbsoluteStandardError],\n            you will probably have to carefully adjust the threshold).\n        n_jobs: Number of parallel jobs to use.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    if isinstance(sampler, PermutationSampler) and u.cache is None:\n        log.warning(\n            \"PermutationSampler requires caching to be enabled or computation \"\n            \"will be doubled wrt. a 'direct' implementation of permutation MC\"\n        )\n\n    if batch_size != 1:\n        warnings.warn(\n            \"Parameter `batch_size` is for experimental use and will be\"\n            \" removed in future versions\",\n            DeprecationWarning,\n        )\n\n    result = ValuationResult.zeros(\n        algorithm=f\"semivalue-{str(sampler)}-{coefficient.__name__}\",  # type: ignore\n        indices=u.data.indices,\n        data_names=u.data.data_names,\n    )\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n    u = parallel_backend.put(u)\n    correction = parallel_backend.put(\n        lambda n, k: coefficient(n, k) * sampler.weight(n, k)\n    )\n\n    max_workers = parallel_backend.effective_n_jobs(n_jobs)\n    n_submitted_jobs = 2 * max_workers  # number of jobs in the queue\n\n    sampler_it = iter(sampler)\n    pbar = tqdm(disable=not progress, total=100, unit=\"%\")\n\n    with parallel_backend.executor(\n        max_workers=max_workers, cancel_futures=True\n    ) as executor:\n        pending: set[Future] = set()\n        while True:\n            pbar.n = 100 * done.completion()\n            pbar.refresh()\n\n            completed, pending = wait(pending, timeout=1, return_when=FIRST_COMPLETED)\n            for future in completed:\n                processed_future = future_processor(\n                    future.result()\n                )  # List of tuples or\n                for batch_future in processed_future:\n                    if isinstance(batch_future, list):  # Case when batch size is &gt; 1\n                        for idx, marginal_val in batch_future:\n                            result.update(idx, marginal_val)\n                    else:  # Batch size 1\n                        idx, marginal_val = batch_future\n                        result.update(idx, marginal_val)\n                    if done(result):\n                        return result\n\n            # Ensure that we always have n_submitted_jobs running\n            try:\n                while len(pending) &lt; n_submitted_jobs:\n                    samples = tuple(islice(sampler_it, batch_size))\n                    if len(samples) == 0:\n                        raise StopIteration\n\n                    # Filter out samples for indices that have already converged\n                    filtered_samples = samples\n                    if skip_converged and np.count_nonzero(done.converged) &gt; 0:\n                        # TODO: cloudpickle can't pickle result of `filter` on python 3.8\n                        filtered_samples = tuple(\n                            filter(lambda t: not done.converged[t[0]], samples)\n                        )\n\n                    if filtered_samples:\n                        pending.add(\n                            executor.submit(\n                                marginal,\n                                u=u,\n                                coefficient=correction,\n                                samples=filtered_samples,\n                            )\n                        )\n            except StopIteration:\n                if len(pending) == 0:\n                    return result\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_shapley_semivalues","title":"compute_shapley_semivalues","text":"<pre><code>compute_shapley_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes Shapley values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Shapley coefficient. Use compute_shapley_values for a more flexible interface and additional methods, including TMCS.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_shapley_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    \"\"\"Computes Shapley values for a given utility function.\n\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Shapley coefficient. Use\n    [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values]\n    for a more flexible interface and additional methods, including TMCS.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per single parallel job.\n        n_jobs: Number of parallel jobs to use.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    # HACK: cannot infer return type because of useless IndexT, NameT\n    return compute_generic_semivalues(  # type: ignore\n        sampler_t(u.data.indices, seed=seed),\n        u,\n        shapley_coefficient,\n        done,\n        batch_size=batch_size,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n        config=config,\n        progress=progress,\n    )\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_banzhaf_semivalues","title":"compute_banzhaf_semivalues","text":"<pre><code>compute_banzhaf_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes Banzhaf values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Banzhaf coefficient.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_banzhaf_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    \"\"\"Computes Banzhaf values for a given utility function.\n\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Banzhaf coefficient.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per single parallel job.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    # HACK: cannot infer return type because of useless IndexT, NameT\n    return compute_generic_semivalues(  # type: ignore\n        sampler_t(u.data.indices, seed=seed),\n        u,\n        banzhaf_coefficient,\n        done,\n        batch_size=batch_size,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n        config=config,\n        progress=progress,\n    )\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_msr_banzhaf_semivalues","title":"compute_msr_banzhaf_semivalues","text":"<pre><code>compute_msr_banzhaf_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = MSRSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes MSR sampled Banzhaf values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Banzhaf coefficient and MSR sampling.</p> <p>This algorithm works by sampling random subsets and then evaluating the utility on that subset only once. Based on the evaluation and the subset indices, the MSRFutureProcessor then computes the marginal updates like in the paper (Wang et. al.)<sup>3</sup>. Their approach updates the semivalues for all data points every time a new evaluation is computed. This increases sample efficiency compared to normal Monte Carlo updates.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>MSRSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_msr_banzhaf_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = MSRSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    \"\"\"Computes MSR sampled Banzhaf values for a given utility function.\n\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Banzhaf coefficient and MSR sampling.\n\n    This algorithm works by sampling random subsets and then evaluating the utility\n    on that subset only once. Based on the evaluation and the subset indices,\n    the MSRFutureProcessor then computes the marginal updates like in the paper\n    (Wang et. al.)&lt;sup&gt;&lt;a href=\"wang_data_2023\"&gt;3&lt;/a&gt;&lt;/sup&gt;.\n    Their approach updates the semivalues for all data points every time a new evaluation\n    is computed. This increases sample efficiency compared to normal Monte Carlo updates.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per single parallel job.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n    \"\"\"\n    # HACK: cannot infer return type because of useless IndexT, NameT\n    return compute_generic_semivalues(  # type: ignore\n        sampler_t(u.data.indices, seed=seed),\n        u,\n        always_one_coefficient,\n        done,\n        marginal=RawUtility(),\n        future_processor=MSRFutureProcessor(u),\n        batch_size=batch_size,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n        config=config,\n        progress=progress,\n    )\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_beta_shapley_semivalues","title":"compute_beta_shapley_semivalues","text":"<pre><code>compute_beta_shapley_semivalues(\n    u: Utility,\n    *,\n    alpha: float = 1,\n    beta: float = 1,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes Beta Shapley values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Beta Shapley coefficient.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>alpha</code> <p>Alpha parameter of the Beta distribution.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> <code>beta</code> <p>Beta parameter of the Beta distribution.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per (parallelized) task.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_beta_shapley_semivalues(\n    u: Utility,\n    *,\n    alpha: float = 1,\n    beta: float = 1,\n    done: StoppingCriterion,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    \"\"\"Computes Beta Shapley values for a given utility function.\n\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Beta Shapley coefficient.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        alpha: Alpha parameter of the Beta distribution.\n        beta: Beta parameter of the Beta distribution.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per (parallelized) task.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    # HACK: cannot infer return type because of useless IndexT, NameT\n    return compute_generic_semivalues(  # type: ignore\n        sampler_t(u.data.indices, seed=seed),\n        u,\n        beta_coefficient(alpha, beta),\n        done,\n        batch_size=batch_size,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n        config=config,\n        progress=progress,\n    )\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_semivalues","title":"compute_semivalues","text":"<pre><code>compute_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    mode: SemiValueMode = SemiValueMode.Shapley,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    seed: Optional[Seed] = None,\n    **kwargs\n) -&gt; ValuationResult\n</code></pre> <p>Convenience entry point for most common semi-value computations.</p> <p>Deprecation warning</p> <p>This method is deprecated and will be replaced in 0.8.0 by the more general implementation of compute_generic_semivalues. Use compute_shapley_semivalues, compute_banzhaf_semivalues, or compute_beta_shapley_semivalues instead.</p> <p>The modes supported with this interface are the following. For greater flexibility use compute_generic_semivalues directly.</p> <ul> <li>SemiValueMode.Shapley:   Shapley values.</li> <li>SemiValueMode.BetaShapley:   Implements the Beta Shapley semi-value as introduced in   (Kwon and Zou, 2022)<sup>1</sup>.   Pass additional keyword arguments <code>alpha</code> and <code>beta</code> to set the   parameters of the Beta distribution (both default to 1).</li> <li>SemiValueMode.Banzhaf: Implements   the Banzhaf semi-value as introduced in (Wang and Jia, 2022)<sup>1</sup>.</li> </ul> <p>See Data valuation for an overview of valuation.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>mode</code> <p>The semi-value mode to use. See SemiValueMode for a list.</p> <p> TYPE: <code>SemiValueMode</code> DEFAULT: <code>Shapley</code> </p> <code>sampler_t</code> <p>The sampler type to use. See sampler for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per (parallelized) task.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments passed to compute_generic_semivalues.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(target=True, deprecated_in=\"0.7.0\", remove_in=\"0.8.0\")\ndef compute_semivalues(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    mode: SemiValueMode = SemiValueMode.Shapley,\n    sampler_t: Type[StochasticSampler] = PermutationSampler,\n    batch_size: int = 1,\n    n_jobs: int = 1,\n    seed: Optional[Seed] = None,\n    **kwargs,\n) -&gt; ValuationResult:\n    \"\"\"Convenience entry point for most common semi-value computations.\n\n    !!! warning \"Deprecation warning\"\n        This method is deprecated and will be replaced in 0.8.0 by the more\n        general implementation of\n        [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n        Use\n        [compute_shapley_semivalues][pydvl.value.semivalues.compute_shapley_semivalues],\n        [compute_banzhaf_semivalues][pydvl.value.semivalues.compute_banzhaf_semivalues],\n        or\n        [compute_beta_shapley_semivalues][pydvl.value.semivalues.compute_beta_shapley_semivalues]\n        instead.\n\n    The modes supported with this interface are the following. For greater\n    flexibility use\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    directly.\n\n    - [SemiValueMode.Shapley][pydvl.value.semivalues.SemiValueMode]:\n      Shapley values.\n    - [SemiValueMode.BetaShapley][pydvl.value.semivalues.SemiValueMode]:\n      Implements the Beta Shapley semi-value as introduced in\n      (Kwon and Zou, 2022)&lt;sup&gt;&lt;a href=\"#kwon_beta_2022\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n      Pass additional keyword arguments `alpha` and `beta` to set the\n      parameters of the Beta distribution (both default to 1).\n    - [SemiValueMode.Banzhaf][pydvl.value.semivalues.SemiValueMode]: Implements\n      the Banzhaf semi-value as introduced in (Wang and Jia, 2022)&lt;sup&gt;&lt;a\n      href=\"#wang_data_2023\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n\n    See [Data valuation][data-valuation] for an overview of valuation.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        mode: The semi-value mode to use. See\n            [SemiValueMode][pydvl.value.semivalues.SemiValueMode] for a list.\n        sampler_t: The sampler type to use. See [sampler][pydvl.value.sampler]\n            for a list.\n        batch_size: Number of marginal evaluations per (parallelized) task.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        kwargs: Additional keyword arguments passed to\n            [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n\n    Returns:\n        Object with the results.\n\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n    \"\"\"\n    if mode == SemiValueMode.Shapley:\n        coefficient = shapley_coefficient\n    elif mode == SemiValueMode.BetaShapley:\n        alpha = kwargs.pop(\"alpha\", 1)\n        beta = kwargs.pop(\"beta\", 1)\n        coefficient = beta_coefficient(alpha, beta)\n    elif mode == SemiValueMode.Banzhaf:\n        coefficient = banzhaf_coefficient\n    else:\n        raise ValueError(f\"Unknown mode {mode}\")\n    coefficient = cast(SVCoefficient, coefficient)\n\n    # HACK: cannot infer return type because of useless IndexT, NameT\n    return compute_generic_semivalues(  # type: ignore\n        sampler_t(u.data.indices, seed=seed),\n        u,\n        coefficient,\n        done,\n        n_jobs=n_jobs,\n        batch_size=batch_size,\n        **kwargs,\n    )\n</code></pre>"},{"location":"api/pydvl/value/stopping/","title":"Stopping","text":""},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping","title":"pydvl.value.stopping","text":"<p>Stopping criteria for value computations.</p> <p>This module provides a basic set of stopping criteria, like MaxUpdates, MaxTime, or HistoryDeviation among others. These can behave in different ways depending on the context. For example, MaxUpdates limits the number of updates to values, which depending on the algorithm may mean a different number of utility evaluations or imply other computations like solving a linear or quadratic program.</p> <p>Stopping criteria are callables that are evaluated on a ValuationResult and return a Status object. They can be combined using boolean operators.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--how-convergence-is-determined","title":"How convergence is determined","text":"<p>Most stopping criteria keep track of the convergence of each index separately but make global decisions based on the overall convergence of some fraction of all indices. For example, if we have a stopping criterion that checks whether the standard error of 90% of values is below a threshold, then methods will keep updating all indices until 90% of them have converged, irrespective of the quality of the individual estimates, and without freezing updates for indices along the way as values individually attain low standard error.</p> <p>This has some practical implications, because some values do tend to converge sooner than others. For example, assume we use the criterion <code>AbsoluteStandardError(0.02) | MaxUpdates(1000)</code>. Then values close to 0 might be marked as \"converged\" rather quickly because they fulfill the first criterion, say after 20 iterations, despite being poor estimates. Because other indices take much longer to have low standard error and the criterion is a global check, the \"converged\" ones keep being updated and end up being good estimates. In this case, this has been beneficial, but one might not wish for converged values to be updated, if one is sure that the criterion is adequate for individual values.</p> <p>Semi-value methods include a parameter <code>skip_converged</code> that allows to skip the computation of values that have converged. The way to avoid doing this too early is to use a more stringent check, e.g. <code>AbsoluteStandardError(1e-3) | MaxUpdates(1000)</code>. With <code>skip_converged=True</code> this check can still take less time than the first one, despite requiring more iterations for some indices.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--choosing-a-stopping-criterion","title":"Choosing a stopping criterion","text":"<p>The choice of a stopping criterion greatly depends on the algorithm and the context. A safe bet is to combine a MaxUpdates or a MaxTime with a HistoryDeviation or an AbsoluteStandardError. The former will ensure that the computation does not run for too long, while the latter will try to achieve results that are stable enough. Note however that if the threshold is too strict, one will always end up running until a maximum number of iterations or time. Also keep in mind that different values converge at different times, so you might want to use tight thresholds and <code>skip_converged</code> as described above for semi-values.</p> Example <p><pre><code>from pydvl.value import AbsoluteStandardError, MaxUpdates, compute_banzhaf_semivalues\n\nutility = ...  # some utility object\ncriterion = AbsoluteStandardError(threshold=1e-3, burn_in=32) | MaxUpdates(1000)\nvalues = compute_banzhaf_semivalues(\n    utility,\n    criterion,\n    skip_converged=True,  # skip values that have converged (CAREFUL!)\n)\n</code></pre> This will compute the Banzhaf semivalues for <code>utility</code> until either the absolute standard error is below <code>1e-3</code> or <code>1000</code> updates have been performed. The <code>burn_in</code> parameter is used to discard the first <code>32</code> updates from the computation of the standard error. The <code>skip_converged</code> parameter is used to avoid computing more marginals for indices that have converged, which is useful if AbsoluteStandardError is met before MaxUpdates for some indices.</p> <p>Warning</p> <p>Be careful not to reuse the same stopping criterion for different computations. The object has state and will not be reset between calls to value computation methods. If you need to reuse the same criterion, you should create a new instance.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--creating-stopping-criteria","title":"Creating stopping criteria","text":"<p>The easiest way is to declare a function implementing the interface StoppingCriterionCallable and wrap it with make_criterion(). This creates a StoppingCriterion object that can be composed with other stopping criteria.</p> <p>Alternatively, and in particular if reporting of completion is required, one can inherit from this class and implement the abstract methods <code>_check</code> and completion.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--combining-stopping-criteria","title":"Combining stopping criteria","text":"<p>Objects of type StoppingCriterion can be combined with the binary operators <code>&amp;</code> (and), and <code>|</code> (or), following the truth tables of Status. The unary operator <code>~</code> (not) is also supported. See StoppingCriterion for details on how these operations affect the behavior of the stopping criteria.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T. and Jia, R., 2023. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning. In: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics, pp. 6388-6421.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterionCallable","title":"StoppingCriterionCallable","text":"<p>             Bases: <code>Protocol</code></p> <p>Signature for a stopping criterion</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion","title":"StoppingCriterion","text":"<pre><code>StoppingCriterion(modify_result: bool = True)\n</code></pre> <p>             Bases: <code>ABC</code></p> <p>A composable callable object to determine whether a computation must stop.</p> <p>A <code>StoppingCriterion</code> is a callable taking a ValuationResult and returning a Status. It also keeps track of individual convergence of values with converged, and reports the overall completion of the computation with completion.</p> <p>Instances of <code>StoppingCriterion</code> can be composed with the binary operators <code>&amp;</code> (and), and <code>|</code> (or), following the truth tables of Status. The unary operator <code>~</code> (not) is also supported. These boolean operations act according to the following rules:</p> <ul> <li>The results of <code>check()</code> are combined with the operator. See   Status for the truth tables.</li> <li>The results of   converged are combined   with the operator (returning another boolean array).</li> <li>The completion   method returns the min, max, or the complement to 1 of the completions of   the operands, for AND, OR and NOT respectively. This is required for cases   where one of the criteria does not keep track of the convergence of single   values, e.g. MaxUpdates, because   completion by   default returns the mean of the boolean convergence array.</li> </ul>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion--subclassing","title":"Subclassing","text":"<p>Subclassing this class requires implementing a <code>check()</code> method that returns a Status object based on a given ValuationResult. This method should update the attribute <code>_converged</code>, which is a boolean array indicating whether the value for each index has converged. When this does not make sense for a particular stopping criterion, completion should be overridden to provide an overall completion value, since its default implementation attempts to compute the mean of <code>_converged</code>.</p> PARAMETER  DESCRIPTION <code>modify_result</code> <p>If <code>True</code> the status of the input ValuationResult is modified in place after the call.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, modify_result: bool = True):\n    self.modify_result = modify_result\n    self._converged = np.full(0, False)\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.completion","title":"completion","text":"<pre><code>completion() -&gt; float\n</code></pre> <p>Returns a value between 0 and 1 indicating the completion of the computation.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def completion(self) -&gt; float:\n    \"\"\"Returns a value between 0 and 1 indicating the completion of the\n    computation.\n    \"\"\"\n    if self.converged.size == 0:\n        return 0.0\n    return float(np.mean(self.converged).item())\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.AbsoluteStandardError","title":"AbsoluteStandardError","text":"<pre><code>AbsoluteStandardError(\n    threshold: float,\n    fraction: float = 1.0,\n    burn_in: int = 4,\n    modify_result: bool = True,\n)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>Determine convergence based on the standard error of the values.</p> <p>If \\(s_i\\) is the standard error for datum \\(i\\), then this criterion returns Converged if \\(s_i &lt; \\epsilon\\) for all \\(i\\) and a threshold value \\(\\epsilon \\gt 0\\).</p> PARAMETER  DESCRIPTION <code>threshold</code> <p>A value is considered to have converged if the standard error is below this threshold. A way of choosing it is to pick some percentage of the range of the values. For Shapley values this is the difference between the maximum and minimum of the utility function (to see this substitute the maximum and minimum values of the utility into the marginal contribution formula).</p> <p> TYPE: <code>float</code> </p> <code>fraction</code> <p>The fraction of values that must have converged for the criterion to return Converged.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>burn_in</code> <p>The number of iterations to ignore before checking for convergence. This is required because computations typically start with zero variance, as a result of using zeros(). The default is set to an arbitrary minimum which is usually enough but may need to be increased.</p> <p> TYPE: <code>int</code> DEFAULT: <code>4</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(\n    self,\n    threshold: float,\n    fraction: float = 1.0,\n    burn_in: int = 4,\n    modify_result: bool = True,\n):\n    super().__init__(modify_result=modify_result)\n    self.threshold = threshold\n    self.fraction = fraction\n    self.burn_in = burn_in\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.AbsoluteStandardError.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.AbsoluteStandardError.completion","title":"completion","text":"<pre><code>completion() -&gt; float\n</code></pre> <p>Returns a value between 0 and 1 indicating the completion of the computation.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def completion(self) -&gt; float:\n    \"\"\"Returns a value between 0 and 1 indicating the completion of the\n    computation.\n    \"\"\"\n    if self.converged.size == 0:\n        return 0.0\n    return float(np.mean(self.converged).item())\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.AbsoluteStandardError.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxChecks","title":"MaxChecks","text":"<pre><code>MaxChecks(n_checks: Optional[int], modify_result: bool = True)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate as soon as the number of checks exceeds the threshold.</p> <p>A \"check\" is one call to the criterion.</p> PARAMETER  DESCRIPTION <code>n_checks</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_checks: Optional[int], modify_result: bool = True):\n    super().__init__(modify_result=modify_result)\n    if n_checks is not None and n_checks &lt; 1:\n        raise ValueError(\"n_iterations must be at least 1 or None\")\n    self.n_checks = n_checks\n    self._count = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxChecks.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxChecks.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxUpdates","title":"MaxUpdates","text":"<pre><code>MaxUpdates(n_updates: Optional[int], modify_result: bool = True)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate if any number of value updates exceeds or equals the given threshold.</p> <p>Note</p> <p>If you want to ensure that all values have been updated, you probably want MinUpdates instead.</p> <p>This checks the <code>counts</code> field of a ValuationResult, i.e. the number of times that each index has been updated. For powerset samplers, the maximum of this number coincides with the maximum number of subsets sampled. For permutation samplers, it coincides with the number of permutations sampled.</p> PARAMETER  DESCRIPTION <code>n_updates</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_updates: Optional[int], modify_result: bool = True):\n    super().__init__(modify_result=modify_result)\n    if n_updates is not None and n_updates &lt; 1:\n        raise ValueError(\"n_updates must be at least 1 or None\")\n    self.n_updates = n_updates\n    self.last_max = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxUpdates.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxUpdates.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MinUpdates","title":"MinUpdates","text":"<pre><code>MinUpdates(n_updates: Optional[int], modify_result: bool = True)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate as soon as all value updates exceed or equal the given threshold.</p> <p>This checks the <code>counts</code> field of a ValuationResult, i.e. the number of times that each index has been updated. For powerset samplers, the minimum of this number is a lower bound for the number of subsets sampled. For permutation samplers, it lower-bounds the amount of permutations sampled.</p> PARAMETER  DESCRIPTION <code>n_updates</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_updates: Optional[int], modify_result: bool = True):\n    super().__init__(modify_result=modify_result)\n    self.n_updates = n_updates\n    self.last_min = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MinUpdates.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MinUpdates.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxTime","title":"MaxTime","text":"<pre><code>MaxTime(seconds: Optional[float], modify_result: bool = True)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate if the computation time exceeds the given number of seconds.</p> <p>Checks the elapsed time since construction</p> PARAMETER  DESCRIPTION <code>seconds</code> <p>Threshold: The computation is terminated if the elapsed time between object construction and a _check exceeds this value. If <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[float]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, seconds: Optional[float], modify_result: bool = True):\n    super().__init__(modify_result=modify_result)\n    self.max_seconds = seconds or np.inf\n    if self.max_seconds &lt;= 0:\n        raise ValueError(\"Number of seconds for MaxTime must be positive or None\")\n    self.start = time()\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxTime.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxTime.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.HistoryDeviation","title":"HistoryDeviation","text":"<pre><code>HistoryDeviation(\n    n_steps: int,\n    rtol: float,\n    pin_converged: bool = True,\n    modify_result: bool = True,\n)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>A simple check for relative distance to a previous step in the computation.</p> <p>The method used by (Ghorbani and Zou, 2019)<sup>1</sup> computes the relative distances between the current values \\(v_i^t\\) and the values at the previous checkpoint \\(v_i^{t-\\tau}\\). If the sum is below a given threshold, the computation is terminated.</p> \\[\\sum_{i=1}^n \\frac{\\left| v_i^t - v_i^{t-\\tau} \\right|}{v_i^t} &lt; \\epsilon.\\] <p>When the denominator is zero, the summand is set to the value of \\(v_i^{ t-\\tau}\\).</p> <p>This implementation is slightly generalised to allow for different number of updates to individual indices, as happens with powerset samplers instead of permutations. Every subset of indices that is found to converge can be pinned to that state. Once all indices have converged the method has converged.</p> <p>Warning</p> <p>This criterion is meant for the reproduction of the results in the paper, but we do not recommend using it in practice.</p> PARAMETER  DESCRIPTION <code>n_steps</code> <p>Checkpoint values every so many updates and use these saved values to compare.</p> <p> TYPE: <code>int</code> </p> <code>rtol</code> <p>Relative tolerance for convergence (\\(\\epsilon\\) in the formula).</p> <p> TYPE: <code>float</code> </p> <code>pin_converged</code> <p>If <code>True</code>, once an index has converged, it is pinned</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(\n    self,\n    n_steps: int,\n    rtol: float,\n    pin_converged: bool = True,\n    modify_result: bool = True,\n):\n    super().__init__(modify_result=modify_result)\n    if n_steps &lt; 1:\n        raise ValueError(\"n_steps must be at least 1\")\n    if rtol &lt;= 0 or rtol &gt;= 1:\n        raise ValueError(\"rtol must be in (0, 1)\")\n\n    self.n_steps = n_steps\n    self.rtol = rtol\n    self.update_op = np.logical_or if pin_converged else np.logical_and\n    self._memory = None  # type: ignore\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.HistoryDeviation.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.HistoryDeviation.completion","title":"completion","text":"<pre><code>completion() -&gt; float\n</code></pre> <p>Returns a value between 0 and 1 indicating the completion of the computation.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def completion(self) -&gt; float:\n    \"\"\"Returns a value between 0 and 1 indicating the completion of the\n    computation.\n    \"\"\"\n    if self.converged.size == 0:\n        return 0.0\n    return float(np.mean(self.converged).item())\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.HistoryDeviation.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.RankCorrelation","title":"RankCorrelation","text":"<pre><code>RankCorrelation(rtol: float, burn_in: int, modify_result: bool = True)\n</code></pre> <p>             Bases: <code>StoppingCriterion</code></p> <p>A check for stability of Spearman correlation between checks.</p> <p>When the change in rank correlation between two successive iterations is below a given threshold, the computation is terminated. The criterion computes the Spearman correlation between two successive iterations. The Spearman correlation uses the ordering indices of the given values and correlates them. This means it focuses on the order of the elements instead of their exact values. If the order stops changing (meaning the Banzhaf semivalues estimates converge), the criterion stops the algorithm.</p> <p>This criterion is used in (Wang et. al.)<sup>2</sup>.</p> PARAMETER  DESCRIPTION <code>rtol</code> <p>Relative tolerance for convergence (\\(\\epsilon\\) in the formula)</p> <p> TYPE: <code>float</code> </p> <code>modify_result</code> <p>If <code>True</code>, the status of the input ValuationResult is modified in place after the call.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>burn_in</code> <p>The minimum number of iterations before checking for convergence. This is required because the first correlation is meaningless.</p> <p> TYPE: <code>int</code> </p> <p>Added in 0.9.0</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(\n    self,\n    rtol: float,\n    burn_in: int,\n    modify_result: bool = True,\n):\n    super().__init__(modify_result=modify_result)\n    if rtol &lt;= 0 or rtol &gt;= 1:\n        raise ValueError(\"rtol must be in (0, 1)\")\n    self.rtol = rtol\n    self.burn_in = burn_in\n    self._memory: NDArray[np.float_] | None = None\n    self._corr = 0.0\n    self._completion = 0.0\n    self._iterations = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.RankCorrelation.converged","title":"converged  <code>property</code>","text":"<pre><code>converged: NDArray[bool_]\n</code></pre> <p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their <code>check()</code>.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.RankCorrelation.__call__","title":"__call__","text":"<pre><code>__call__(result: ValuationResult) -&gt; Status\n</code></pre> <p>Calls <code>check()</code>, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n    \"\"\"Calls `check()`, maybe updating the result.\"\"\"\n    if len(result) == 0:\n        logger.warning(\n            \"At least one iteration finished but no results where generated. \"\n            \"Please check that your scorer and utility return valid numbers.\"\n        )\n    status = self._check(result)\n    if self.modify_result:  # FIXME: this is not nice\n        result._status = status\n    return status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.make_criterion","title":"make_criterion","text":"<pre><code>make_criterion(\n    fun: StoppingCriterionCallable,\n    converged: Callable[[], NDArray[bool_]] | None = None,\n    completion: Callable[[], float] | None = None,\n    name: str | None = None,\n) -&gt; Type[StoppingCriterion]\n</code></pre> <p>Create a new StoppingCriterion from a function. Use this to enable simpler functions to be composed with bitwise operators</p> PARAMETER  DESCRIPTION <code>fun</code> <p>The callable to wrap.</p> <p> TYPE: <code>StoppingCriterionCallable</code> </p> <code>converged</code> <p>A callable that returns a boolean array indicating what values have converged.</p> <p> TYPE: <code>Callable[[], NDArray[bool_]] | None</code> DEFAULT: <code>None</code> </p> <code>completion</code> <p>A callable that returns a value between 0 and 1 indicating the rate of completion of the computation. If not provided, the fraction of converged values is used.</p> <p> TYPE: <code>Callable[[], float] | None</code> DEFAULT: <code>None</code> </p> <code>name</code> <p>The name of the new criterion. If <code>None</code>, the <code>__name__</code> of the function is used.</p> <p> TYPE: <code>str | None</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Type[StoppingCriterion]</code> <p>A new subclass of StoppingCriterion.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def make_criterion(\n    fun: StoppingCriterionCallable,\n    converged: Callable[[], NDArray[np.bool_]] | None = None,\n    completion: Callable[[], float] | None = None,\n    name: str | None = None,\n) -&gt; Type[StoppingCriterion]:\n    \"\"\"Create a new [StoppingCriterion][pydvl.value.stopping.StoppingCriterion] from a function.\n    Use this to enable simpler functions to be composed with bitwise operators\n\n    Args:\n        fun: The callable to wrap.\n        converged: A callable that returns a boolean array indicating what\n            values have converged.\n        completion: A callable that returns a value between 0 and 1 indicating\n            the rate of completion of the computation. If not provided, the fraction\n            of converged values is used.\n        name: The name of the new criterion. If `None`, the `__name__` of\n            the function is used.\n\n    Returns:\n        A new subclass of [StoppingCriterion][pydvl.value.stopping.StoppingCriterion].\n    \"\"\"\n\n    class WrappedCriterion(StoppingCriterion):\n        def __init__(self, modify_result: bool = True):\n            super().__init__(modify_result=modify_result)\n            self._name = name or getattr(fun, \"__name__\", \"WrappedCriterion\")\n\n        def _check(self, result: ValuationResult) -&gt; Status:\n            return fun(result)\n\n        @property\n        def converged(self) -&gt; NDArray[np.bool_]:\n            if converged is None:\n                return super().converged\n            return converged()\n\n        def __str__(self):\n            return self._name\n\n        def completion(self) -&gt; float:\n            if completion is None:\n                return super().completion()\n            return completion()\n\n    return WrappedCriterion\n</code></pre>"},{"location":"api/pydvl/value/least_core/","title":"Least core","text":""},{"location":"api/pydvl/value/least_core/#pydvl.value.least_core","title":"pydvl.value.least_core","text":"<p>New in version 0.4.0</p> <p>This package holds all routines for the computation of Least Core data values.</p> <p>Please refer to Data valuation for an overview.</p> <p>In addition to the standard interface via compute_least_core_values(), because computing the Least Core values requires the solution of a linear and a quadratic problem after computing all the utility values, there is the possibility of performing each step separately. This is useful when running multiple experiments: use lc_prepare_problem() or mclc_prepare_problem() to prepare a list of problems to solve, then solve them in parallel with lc_solve_problems().</p> <p>Note that mclc_prepare_problem() is parallelized itself, so preparing the problems should be done in sequence in this case. The solution of the linear systems can then be done in parallel.</p>"},{"location":"api/pydvl/value/least_core/#pydvl.value.least_core.LeastCoreMode","title":"LeastCoreMode","text":"<p>             Bases: <code>Enum</code></p> <p>Available Least Core algorithms.</p>"},{"location":"api/pydvl/value/least_core/#pydvl.value.least_core.compute_least_core_values","title":"compute_least_core_values","text":"<pre><code>compute_least_core_values(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    n_iterations: Optional[int] = None,\n    mode: LeastCoreMode = LeastCoreMode.MonteCarlo,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = False,\n    **kwargs\n) -&gt; ValuationResult\n</code></pre> <p>Umbrella method to compute Least Core values with any of the available algorithms.</p> <p>See Data valuation for an overview.</p> <p>The following algorithms are available. Note that the exact method can only work with very small datasets and is thus intended only for testing.</p> <ul> <li><code>exact</code>: uses the complete powerset of the training set for the constraints   combinatorial_exact_shapley().</li> <li><code>montecarlo</code>:  uses the approximate Monte Carlo Least Core algorithm.   Implemented in montecarlo_least_core().</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_jobs</code> <p>Number of jobs to run in parallel. Only used for Monte Carlo Least Core.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_iterations</code> <p>Number of subsets to sample and evaluate the utility on. Only used for Monte Carlo Least Core.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>Algorithm to use. See LeastCoreMode for available options.</p> <p> TYPE: <code>LeastCoreMode</code> DEFAULT: <code>MonteCarlo</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>solver_options</code> <p>Optional dictionary of options passed to the solvers.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the computed values.</p> <p>New in version 0.5.0</p> Source code in <code>src/pydvl/value/least_core/__init__.py</code> <pre><code>def compute_least_core_values(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    n_iterations: Optional[int] = None,\n    mode: LeastCoreMode = LeastCoreMode.MonteCarlo,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = False,\n    **kwargs,\n) -&gt; ValuationResult:\n    \"\"\"Umbrella method to compute Least Core values with any of the available\n    algorithms.\n\n    See [Data valuation][data-valuation] for an overview.\n\n    The following algorithms are available. Note that the exact method can only\n    work with very small datasets and is thus intended only for testing.\n\n    - `exact`: uses the complete powerset of the training set for the constraints\n      [combinatorial_exact_shapley()][pydvl.value.shapley.naive.combinatorial_exact_shapley].\n    - `montecarlo`:  uses the approximate Monte Carlo Least Core algorithm.\n      Implemented in [montecarlo_least_core()][pydvl.value.least_core.montecarlo.montecarlo_least_core].\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_jobs: Number of jobs to run in parallel. Only used for Monte Carlo\n            Least Core.\n        n_iterations: Number of subsets to sample and evaluate the utility on.\n            Only used for Monte Carlo Least Core.\n        mode: Algorithm to use. See\n            [LeastCoreMode][pydvl.value.least_core.LeastCoreMode] for available\n            options.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Optional dictionary of options passed to the solvers.\n\n    Returns:\n        Object with the computed values.\n\n    !!! tip \"New in version 0.5.0\"\n    \"\"\"\n\n    if mode == LeastCoreMode.MonteCarlo:\n        # TODO fix progress showing in remote case\n        progress = False\n        if n_iterations is None:\n            raise ValueError(\"n_iterations cannot be None for Monte Carlo Least Core\")\n        return montecarlo_least_core(  # type: ignore\n            u=u,\n            n_iterations=n_iterations,\n            n_jobs=n_jobs,\n            progress=progress,\n            non_negative_subsidy=non_negative_subsidy,\n            solver_options=solver_options,\n            **kwargs,\n        )\n    elif mode == LeastCoreMode.Exact:\n        return exact_least_core(\n            u=u,\n            progress=progress,\n            non_negative_subsidy=non_negative_subsidy,\n            solver_options=solver_options,\n        )\n\n    raise ValueError(f\"Invalid value encountered in {mode=}\")\n</code></pre>"},{"location":"api/pydvl/value/least_core/common/","title":"Common","text":""},{"location":"api/pydvl/value/least_core/common/#pydvl.value.least_core.common","title":"pydvl.value.least_core.common","text":""},{"location":"api/pydvl/value/least_core/common/#pydvl.value.least_core.common.lc_solve_problem","title":"lc_solve_problem","text":"<pre><code>lc_solve_problem(\n    problem: LeastCoreProblem,\n    *,\n    u: Utility,\n    algorithm: str,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None\n) -&gt; ValuationResult\n</code></pre> <p>Solves a linear problem as prepared by mclc_prepare_problem(). Useful for parallel execution of multiple experiments by running this as a remote task.</p> <p>See exact_least_core() or montecarlo_least_core() for argument descriptions.</p> Source code in <code>src/pydvl/value/least_core/common.py</code> <pre><code>def lc_solve_problem(\n    problem: LeastCoreProblem,\n    *,\n    u: Utility,\n    algorithm: str,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n) -&gt; ValuationResult:\n    \"\"\"Solves a linear problem as prepared by\n    [mclc_prepare_problem()][pydvl.value.least_core.montecarlo.mclc_prepare_problem].\n    Useful for parallel execution of multiple experiments by running this as a\n    remote task.\n\n    See [exact_least_core()][pydvl.value.least_core.naive.exact_least_core] or\n    [montecarlo_least_core()][pydvl.value.least_core.montecarlo.montecarlo_least_core] for\n    argument descriptions.\n    \"\"\"\n    n = len(u.data)\n\n    if np.any(np.isnan(problem.utility_values)):\n        warnings.warn(\n            f\"Calculation returned \"\n            f\"{np.sum(np.isnan(problem.utility_values))} NaN \"\n            f\"values out of {problem.utility_values.size}\",\n            RuntimeWarning,\n        )\n\n    if solver_options is None:\n        solver_options = {}\n\n    if \"solver\" not in solver_options:\n        solver_options[\"solver\"] = cp.SCS\n\n    if \"max_iters\" not in solver_options and solver_options[\"solver\"] == cp.SCS:\n        solver_options[\"max_iters\"] = 10000\n\n    logger.debug(\"Removing possible duplicate values in lower bound array\")\n    b_lb = problem.utility_values\n    A_lb, unique_indices = np.unique(problem.A_lb, return_index=True, axis=0)\n    b_lb = b_lb[unique_indices]\n\n    logger.debug(\"Building equality constraint\")\n    A_eq = np.ones((1, n))\n    # We might have already computed the total utility one or more times.\n    # This is the index of the row(s) in A_lb with all ones.\n    total_utility_indices = np.where(A_lb.sum(axis=1) == n)[0]\n    if len(total_utility_indices) == 0:\n        b_eq = np.array([u(u.data.indices)])\n    else:\n        b_eq = b_lb[total_utility_indices]\n        # Remove the row(s) corresponding to the total utility\n        # from the lower bound constraints\n        # because given the equality constraint\n        # it is the same as using the constraint e &gt;= 0\n        # (i.e. setting non_negative_subsidy = True).\n        mask: NDArray[np.bool_] = np.ones_like(b_lb, dtype=bool)\n        mask[total_utility_indices] = False\n        b_lb = b_lb[mask]\n        A_lb = A_lb[mask]\n\n    # Remove the row(s) corresponding to the empty subset\n    # because, given u(\u2205) = (which is almost always the case,\n    # it is the same as using the constraint e &gt;= 0\n    # (i.e. setting non_negative_subsidy = True).\n    emptyset_utility_indices = np.where(A_lb.sum(axis=1) == 0)[0]\n    if len(emptyset_utility_indices) &gt; 0:\n        mask = np.ones_like(b_lb, dtype=bool)\n        mask[emptyset_utility_indices] = False\n        b_lb = b_lb[mask]\n        A_lb = A_lb[mask]\n\n    _, subsidy = _solve_least_core_linear_program(\n        A_eq=A_eq,\n        b_eq=b_eq,\n        A_lb=A_lb,\n        b_lb=b_lb,\n        non_negative_subsidy=non_negative_subsidy,\n        solver_options=solver_options,\n    )\n\n    values: Optional[NDArray[np.float_]]\n\n    if subsidy is None:\n        logger.debug(\"No values were found\")\n        status = Status.Failed\n        values = np.empty(n)\n        values[:] = np.nan\n        subsidy = np.nan\n    else:\n        values = _solve_egalitarian_least_core_quadratic_program(\n            subsidy,\n            A_eq=A_eq,\n            b_eq=b_eq,\n            A_lb=A_lb,\n            b_lb=b_lb,\n            solver_options=solver_options,\n        )\n\n        if values is None:\n            logger.debug(\"No values were found\")\n            status = Status.Failed\n            values = np.empty(n)\n            values[:] = np.nan\n            subsidy = np.nan\n        else:\n            status = Status.Converged\n\n    return ValuationResult(\n        algorithm=algorithm,\n        status=status,\n        values=values,\n        subsidy=subsidy,\n        stderr=None,\n        data_names=u.data.data_names,\n    )\n</code></pre>"},{"location":"api/pydvl/value/least_core/common/#pydvl.value.least_core.common.lc_solve_problems","title":"lc_solve_problems","text":"<pre><code>lc_solve_problems(\n    problems: Sequence[LeastCoreProblem],\n    u: Utility,\n    algorithm: str,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    n_jobs: int = 1,\n    non_negative_subsidy: bool = True,\n    solver_options: Optional[dict] = None,\n    **options\n) -&gt; List[ValuationResult]\n</code></pre> <p>Solves a list of linear problems in parallel.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility.</p> <p> TYPE: <code>Utility</code> </p> <code>problems</code> <p>Least Core problems to solve, as returned by mclc_prepare_problem().</p> <p> TYPE: <code>Sequence[LeastCoreProblem]</code> </p> <code>algorithm</code> <p>Name of the valuation algorithm.</p> <p> TYPE: <code>str</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to run.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>solver_options</code> <p>Additional options to pass to the solver.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>List[ValuationResult]</code> <p>List of solutions.</p> Source code in <code>src/pydvl/value/least_core/common.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef lc_solve_problems(\n    problems: Sequence[LeastCoreProblem],\n    u: Utility,\n    algorithm: str,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    n_jobs: int = 1,\n    non_negative_subsidy: bool = True,\n    solver_options: Optional[dict] = None,\n    **options,\n) -&gt; List[ValuationResult]:\n    \"\"\"Solves a list of linear problems in parallel.\n\n    Args:\n        u: Utility.\n        problems: Least Core problems to solve, as returned by\n            [mclc_prepare_problem()][pydvl.value.least_core.montecarlo.mclc_prepare_problem].\n        algorithm: Name of the valuation algorithm.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        n_jobs: Number of parallel jobs to run.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Additional options to pass to the solver.\n\n    Returns:\n        List of solutions.\n    \"\"\"\n\n    def _map_func(\n        problems: List[LeastCoreProblem], *args, **kwargs\n    ) -&gt; List[ValuationResult]:\n        return [lc_solve_problem(p, *args, **kwargs) for p in problems]\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    map_reduce_job: MapReduceJob[\n        \"LeastCoreProblem\", \"List[ValuationResult]\"\n    ] = MapReduceJob(\n        inputs=problems,\n        map_func=_map_func,\n        map_kwargs=dict(\n            u=u,\n            algorithm=algorithm,\n            non_negative_subsidy=non_negative_subsidy,\n            solver_options=solver_options,\n            **options,\n        ),\n        reduce_func=lambda x: list(itertools.chain(*x)),\n        parallel_backend=parallel_backend,\n        n_jobs=n_jobs,\n    )\n    solutions = map_reduce_job()\n\n    return solutions\n</code></pre>"},{"location":"api/pydvl/value/least_core/montecarlo/","title":"Montecarlo","text":""},{"location":"api/pydvl/value/least_core/montecarlo/#pydvl.value.least_core.montecarlo","title":"pydvl.value.least_core.montecarlo","text":""},{"location":"api/pydvl/value/least_core/montecarlo/#pydvl.value.least_core.montecarlo.montecarlo_least_core","title":"montecarlo_least_core","text":"<pre><code>montecarlo_least_core(\n    u: Utility,\n    n_iterations: int,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes approximate Least Core values using a Monte Carlo approach.</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\ &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp; , \\forall S \\in \\{S_1, S_2, \\dots, S_m \\overset{\\mathrm{iid}}{\\sim} U(2^N) \\} \\end{array} \\] <p>Where:</p> <ul> <li>\\(U(2^N)\\) is the uniform distribution over the powerset of \\(N\\).</li> <li>\\(m\\) is the number of subsets that will be sampled and whose utility will   be computed and used to compute the data values.</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_iterations</code> <p>total number of iterations to use</p> <p> TYPE: <code>int</code> </p> <code>n_jobs</code> <p>number of jobs across which to distribute the computation</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>solver_options</code> <p>Dictionary of options that will be used to select a solver and to configure it. Refer to cvxpy's documentation for all possible options.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, shows a tqdm progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values and the least core value.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/least_core/montecarlo.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef montecarlo_least_core(\n    u: Utility,\n    n_iterations: int,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    r\"\"\"Computes approximate Least Core values using a Monte Carlo approach.\n\n    $$\n    \\begin{array}{lll}\n    \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\\n    \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\\n    &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp; ,\n    \\forall S \\in \\{S_1, S_2, \\dots, S_m \\overset{\\mathrm{iid}}{\\sim} U(2^N) \\}\n    \\end{array}\n    $$\n\n    Where:\n\n    * $U(2^N)$ is the uniform distribution over the powerset of $N$.\n    * $m$ is the number of subsets that will be sampled and whose utility will\n      be computed and used to compute the data values.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_iterations: total number of iterations to use\n        n_jobs: number of jobs across which to distribute the computation\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Dictionary of options that will be used to select a solver\n            and to configure it. Refer to [cvxpy's\n            documentation](https://www.cvxpy.org/tutorial/advanced/index.html#setting-solver-options)\n            for all possible options.\n        progress: If True, shows a tqdm progress bar\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Object with the data values and the least core value.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    problem = mclc_prepare_problem(\n        u,\n        n_iterations,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n        config=config,\n        progress=progress,\n        seed=seed,\n    )\n    return lc_solve_problem(\n        problem,\n        u=u,\n        algorithm=\"montecarlo_least_core\",\n        non_negative_subsidy=non_negative_subsidy,\n        solver_options=solver_options,\n    )\n</code></pre>"},{"location":"api/pydvl/value/least_core/montecarlo/#pydvl.value.least_core.montecarlo.mclc_prepare_problem","title":"mclc_prepare_problem","text":"<pre><code>mclc_prepare_problem(\n    u: Utility,\n    n_iterations: int,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; LeastCoreProblem\n</code></pre> <p>Prepares a linear problem by sampling subsets of the data. Use this to separate the problem preparation from the solving with lc_solve_problem(). Useful for parallel execution of multiple experiments.</p> <p>See montecarlo_least_core for argument descriptions.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/least_core/montecarlo.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef mclc_prepare_problem(\n    u: Utility,\n    n_iterations: int,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; LeastCoreProblem:\n    \"\"\"Prepares a linear problem by sampling subsets of the data. Use this to\n    separate the problem preparation from the solving with\n    [lc_solve_problem()][pydvl.value.least_core.common.lc_solve_problem]. Useful\n    for parallel execution of multiple experiments.\n\n    See\n    [montecarlo_least_core][pydvl.value.least_core.montecarlo.montecarlo_least_core]\n    for argument descriptions.\n\n    !!! note \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    n = len(u.data)\n\n    if n_iterations &lt; n:\n        warnings.warn(\n            f\"Number of iterations '{n_iterations}' is smaller the size of the dataset '{n}'. \"\n            f\"This is not optimal because in the worst case we need at least '{n}' constraints \"\n            \"to satisfy the individual rationality condition.\"\n        )\n\n    if n_iterations &gt; 2**n:\n        warnings.warn(\n            f\"Passed n_iterations is greater than the number subsets! \"\n            f\"Setting it to 2^{n}\",\n            RuntimeWarning,\n        )\n        n_iterations = 2**n\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    iterations_per_job = max(\n        1, n_iterations // parallel_backend.effective_n_jobs(n_jobs)\n    )\n\n    map_reduce_job: MapReduceJob[\"Utility\", \"LeastCoreProblem\"] = MapReduceJob(\n        inputs=u,\n        map_func=_montecarlo_least_core,\n        reduce_func=_reduce_func,\n        map_kwargs=dict(n_iterations=iterations_per_job, progress=progress),\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n    )\n\n    return map_reduce_job(seed=seed)\n</code></pre>"},{"location":"api/pydvl/value/least_core/naive/","title":"Naive","text":""},{"location":"api/pydvl/value/least_core/naive/#pydvl.value.least_core.naive","title":"pydvl.value.least_core.naive","text":""},{"location":"api/pydvl/value/least_core/naive/#pydvl.value.least_core.naive.exact_least_core","title":"exact_least_core","text":"<pre><code>exact_least_core(\n    u: Utility,\n    *,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = True\n) -&gt; ValuationResult\n</code></pre> <p>Computes the exact Least Core values.</p> <p>Note</p> <p>If the training set contains more than 20 instances a warning is printed because the computation is very expensive. This method is mostly used for internal testing and simple use cases. Please refer to the Monte Carlo method for practical applications.</p> <p>The least core is the solution to the following Linear Programming problem:</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\ &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp;, \\forall S \\subseteq N \\\\ \\end{array} \\] <p>Where \\(N = \\{1, 2, \\dots, n\\}\\) are the training set's indices.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>solver_options</code> <p>Dictionary of options that will be used to select a solver and to configure it. Refer to the cvxpy's documentation for all possible options.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, shows a tqdm progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values and the least core value.</p> Source code in <code>src/pydvl/value/least_core/naive.py</code> <pre><code>def exact_least_core(\n    u: Utility,\n    *,\n    non_negative_subsidy: bool = False,\n    solver_options: Optional[dict] = None,\n    progress: bool = True,\n) -&gt; ValuationResult:\n    r\"\"\"Computes the exact Least Core values.\n\n    !!! Note\n        If the training set contains more than 20 instances a warning is printed\n        because the computation is very expensive. This method is mostly used for\n        internal testing and simple use cases. Please refer to the\n        [Monte Carlo method][pydvl.value.least_core.montecarlo.montecarlo_least_core]\n        for practical applications.\n\n    The least core is the solution to the following Linear Programming problem:\n\n    $$\n    \\begin{array}{lll}\n    \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\\n    \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\\n    &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp;, \\forall S \\subseteq N \\\\\n    \\end{array}\n    $$\n\n    Where $N = \\{1, 2, \\dots, n\\}$ are the training set's indices.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Dictionary of options that will be used to select a solver\n            and to configure it. Refer to the [cvxpy's\n            documentation](https://www.cvxpy.org/tutorial/advanced/index.html#setting-solver-options)\n            for all possible options.\n        progress: If True, shows a tqdm progress bar\n\n    Returns:\n        Object with the data values and the least core value.\n    \"\"\"\n    n = len(u.data)\n    if n &gt; 20:  # Arbitrary choice, will depend on time required, caching, etc.\n        warnings.warn(f\"Large dataset! Computation requires 2^{n} calls to model.fit()\")\n\n    problem = lc_prepare_problem(u, progress=progress)\n    return lc_solve_problem(\n        problem=problem,\n        u=u,\n        algorithm=\"exact_least_core\",\n        non_negative_subsidy=non_negative_subsidy,\n        solver_options=solver_options,\n    )\n</code></pre>"},{"location":"api/pydvl/value/least_core/naive/#pydvl.value.least_core.naive.lc_prepare_problem","title":"lc_prepare_problem","text":"<pre><code>lc_prepare_problem(u: Utility, progress: bool = False) -&gt; LeastCoreProblem\n</code></pre> <p>Prepares a linear problem with all subsets of the data Use this to separate the problem preparation from the solving with lc_solve_problem(). Useful for parallel execution of multiple experiments.</p> <p>See exact_least_core() for argument descriptions.</p> Source code in <code>src/pydvl/value/least_core/naive.py</code> <pre><code>def lc_prepare_problem(u: Utility, progress: bool = False) -&gt; LeastCoreProblem:\n    \"\"\"Prepares a linear problem with all subsets of the data\n    Use this to separate the problem preparation from the solving with\n    [lc_solve_problem()][pydvl.value.least_core.common.lc_solve_problem]. Useful for\n    parallel execution of multiple experiments.\n\n    See [exact_least_core()][pydvl.value.least_core.naive.exact_least_core] for argument\n    descriptions.\n    \"\"\"\n    n = len(u.data)\n\n    logger.debug(\"Building vectors and matrices for linear programming problem\")\n    powerset_size = 2**n\n    A_lb = np.zeros((powerset_size, n))\n\n    logger.debug(\"Iterating over all subsets\")\n    utility_values = np.zeros(powerset_size)\n    for i, subset in enumerate(  # type: ignore\n        tqdm(\n            powerset(u.data.indices),\n            disable=not progress,\n            total=powerset_size - 1,\n            position=0,\n        )\n    ):\n        indices: NDArray[np.bool_] = np.zeros(n, dtype=bool)\n        indices[list(subset)] = True\n        A_lb[i, indices] = 1\n        utility_values[i] = u(subset)  # type: ignore\n\n    return LeastCoreProblem(utility_values, A_lb)\n</code></pre>"},{"location":"api/pydvl/value/loo/","title":"Loo","text":""},{"location":"api/pydvl/value/loo/#pydvl.value.loo","title":"pydvl.value.loo","text":""},{"location":"api/pydvl/value/loo/loo/","title":"Loo","text":""},{"location":"api/pydvl/value/loo/loo/#pydvl.value.loo.loo","title":"pydvl.value.loo.loo","text":""},{"location":"api/pydvl/value/loo/loo/#pydvl.value.loo.loo.compute_loo","title":"compute_loo","text":"<pre><code>compute_loo(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = True\n) -&gt; ValuationResult\n</code></pre> <p>Computes leave one out value:</p> \\[v(i) = u(D) - u(D \\setminus \\{i\\}) \\] PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>If True, display a progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, display a progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.7.0</p> <p>Renamed from <code>naive_loo</code> and added parallel computation.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/loo/loo.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_loo(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = True,\n) -&gt; ValuationResult:\n    r\"\"\"Computes leave one out value:\n\n    $$v(i) = u(D) - u(D \\setminus \\{i\\}) $$\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        progress: If True, display a progress bar\n        n_jobs: Number of parallel jobs to use\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: If True, display a progress bar\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"New in version 0.7.0\"\n        Renamed from `naive_loo` and added parallel computation.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    if len(u.data) &lt; 3:\n        raise ValueError(\"Dataset must have at least 2 elements\")\n\n    result = ValuationResult.zeros(\n        algorithm=\"loo\",\n        indices=u.data.indices,\n        data_names=u.data.data_names,\n    )\n\n    all_indices = set(u.data.indices)\n    total_utility = u(u.data.indices)\n\n    def fun(idx: int) -&gt; tuple[int, float]:\n        return idx, total_utility - u(all_indices.difference({idx}))\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n    max_workers = parallel_backend.effective_n_jobs(n_jobs)\n    n_submitted_jobs = 2 * max_workers  # number of jobs in the queue\n\n    # NOTE: this could be done with a simple executor.map(), but we want to\n    # display a progress bar\n\n    with parallel_backend.executor(\n        max_workers=max_workers, cancel_futures=True\n    ) as executor:\n        pending: set[Future] = set()\n        index_it = iter(u.data.indices)\n\n        pbar = tqdm(disable=not progress, total=100, unit=\"%\")\n        while True:\n            pbar.n = 100 * sum(result.counts) / len(u.data)\n            pbar.refresh()\n            completed, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED)\n            for future in completed:\n                idx, marginal = future.result()\n                result.update(idx, marginal)\n\n            # Ensure that we always have n_submitted_jobs running\n            try:\n                for _ in range(n_submitted_jobs - len(pending)):\n                    pending.add(executor.submit(fun, next(index_it)))\n            except StopIteration:\n                if len(pending) == 0:\n                    return result\n</code></pre>"},{"location":"api/pydvl/value/oob/","title":"Oob","text":""},{"location":"api/pydvl/value/oob/#pydvl.value.oob","title":"pydvl.value.oob","text":""},{"location":"api/pydvl/value/oob/oob/","title":"Oob","text":""},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob","title":"pydvl.value.oob.oob","text":""},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob--references","title":"References","text":"<ol> <li> <p>Kwon et al. Data-OOB: Out-of-bag Estimate as a Simple and Efficient Data Value. In: Published at ICML 2023\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.compute_data_oob","title":"compute_data_oob","text":"<pre><code>compute_data_oob(\n    u: Utility,\n    *,\n    n_est: int = 10,\n    max_samples: float = 0.8,\n    loss: Optional[LossFunction] = None,\n    n_jobs: Optional[int] = None,\n    seed: Optional[Seed] = None,\n    progress: bool = False\n) -&gt; ValuationResult\n</code></pre> <p>Computes Data out of bag values</p> <p>This implements the method described in (Kwon and Zou, 2023)<sup>1</sup>. It fits several base estimators provided through u.model through a bagging process. The point value corresponds to the average loss of estimators which were not fit on it.</p> <p>\\(w_{bj}\\in Z\\) is the number of times the j-th datum \\((x_j, y_j)\\) is selected in the b-th bootstrap dataset.</p> \\[\\psi((x_i,y_i),\\Theta_B):=\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i, \\hat{f}_b(x_i))}{\\sum_{b=1}^{B} \\mathbb{1} (w_{bi}=0)}\\] <p>With:</p> \\[ T: Y \\times Y \\rightarrow \\mathbb{R} \\] <p>T is a score function that represents the goodness of a weak learner \\(\\hat{f}_b\\) at the i-th datum \\((x_i, y_i)\\).</p> <p><code>n_est</code> and <code>max_samples</code> must be tuned jointly to ensure that all samples are at least 1 time out-of-bag, otherwise the result could include a NaN value for that datum.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>n_est</code> <p>Number of estimator used in the bagging procedure.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>max_samples</code> <p>The fraction of samples to draw to train each base estimator.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>loss</code> <p>A function taking as parameters model prediction and corresponding data labels(y_true, y_pred) and returning an array of point-wise errors.</p> <p> TYPE: <code>Optional[LossFunction]</code> DEFAULT: <code>None</code> </p> <code>n_jobs</code> <p>The number of jobs to run in parallel used in the bagging procedure for both fit and predict.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def compute_data_oob(\n    u: Utility,\n    *,\n    n_est: int = 10,\n    max_samples: float = 0.8,\n    loss: Optional[LossFunction] = None,\n    n_jobs: Optional[int] = None,\n    seed: Optional[Seed] = None,\n    progress: bool = False,\n) -&gt; ValuationResult:\n    r\"\"\"Computes Data out of bag values\n\n    This implements the method described in\n    (Kwon and Zou, 2023)&lt;sup&gt;&lt;a href=\"kwon_data_2023\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    It fits several base estimators provided through u.model through a bagging\n    process. The point value corresponds to the average loss of estimators which\n    were not fit on it.\n\n    $w_{bj}\\in Z$ is the number of times the j-th datum $(x_j, y_j)$ is selected\n    in the b-th bootstrap dataset.\n\n    $$\\psi((x_i,y_i),\\Theta_B):=\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i,\n    \\hat{f}_b(x_i))}{\\sum_{b=1}^{B}\n    \\mathbb{1}\n    (w_{bi}=0)}$$\n\n    With:\n\n    $$\n    T: Y \\times Y\n    \\rightarrow \\mathbb{R}\n    $$\n\n    T is a score function that represents the goodness of a weak learner\n    $\\hat{f}_b$ at the i-th datum $(x_i, y_i)$.\n\n    `n_est` and `max_samples` must be tuned jointly to ensure that all samples\n    are at least 1 time out-of-bag, otherwise the result could include a NaN\n    value for that datum.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        n_est: Number of estimator used in the bagging procedure.\n        max_samples: The fraction of samples to draw to train each base\n            estimator.\n        loss: A function taking as parameters model prediction and corresponding\n            data labels(y_true, y_pred) and returning an array of point-wise errors.\n        n_jobs: The number of jobs to run in parallel used in the bagging\n            procedure for both fit and predict.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        progress: If True, display a progress bar.\n\n    Returns:\n        Object with the data values.\n    \"\"\"\n    rng = np.random.default_rng(seed)\n    random_state = np.random.RandomState(rng.bit_generator)\n\n    result: ValuationResult[np.int_, np.object_] = ValuationResult.empty(\n        algorithm=\"data_oob\", indices=u.data.indices, data_names=u.data.data_names\n    )\n\n    if is_classifier(u.model):\n        bag = BaggingClassifier(\n            u.model,\n            n_estimators=n_est,\n            max_samples=max_samples,\n            n_jobs=n_jobs,\n            random_state=random_state,\n        )\n        if loss is None:\n            loss = point_wise_accuracy\n    elif is_regressor(u.model):\n        bag = BaggingRegressor(\n            u.model,\n            n_estimators=n_est,\n            max_samples=max_samples,\n            n_jobs=n_jobs,\n            random_state=random_state,\n        )\n        if loss is None:\n            loss = neg_l2_distance\n    else:\n        raise Exception(\n            \"Model has to be a classifier or a regressor in sklearn format.\"\n        )\n\n    bag.fit(u.data.x_train, u.data.y_train)\n\n    for est, samples in tqdm(\n        zip(bag.estimators_, bag.estimators_samples_), disable=not progress, total=n_est\n    ):  # The bottleneck is the bag fitting not this part so TQDM is not very useful here\n        oob_idx = np.setxor1d(u.data.indices, np.unique(samples))\n        array_loss = loss(\n            y_true=u.data.y_train[oob_idx],\n            y_pred=est.predict(u.data.x_train[oob_idx]),\n        )\n        result += ValuationResult(\n            algorithm=\"data_oob\",\n            indices=oob_idx,\n            values=array_loss,\n            counts=np.ones_like(array_loss, dtype=u.data.indices.dtype),\n        )\n    return result\n</code></pre>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.point_wise_accuracy","title":"point_wise_accuracy","text":"<pre><code>point_wise_accuracy(y_true: NDArray[T], y_pred: NDArray[T]) -&gt; NDArray[T]\n</code></pre> <p>Point-wise 0-1 loss between two arrays</p> PARAMETER  DESCRIPTION <code>y_true</code> <p>Array of true values (e.g. labels)</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>y_pred</code> <p>Array of estimated values (e.g. model predictions)</p> <p> TYPE: <code>NDArray[T]</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>Array with point-wise 0-1 losses between labels and model predictions</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def point_wise_accuracy(y_true: NDArray[T], y_pred: NDArray[T]) -&gt; NDArray[T]:\n    r\"\"\"Point-wise 0-1 loss between two arrays\n\n    Args:\n        y_true: Array of true values (e.g. labels)\n        y_pred: Array of estimated values (e.g. model predictions)\n\n    Returns:\n        Array with point-wise 0-1 losses between labels and model predictions\n    \"\"\"\n    return np.array(y_pred == y_true, dtype=y_pred.dtype)\n</code></pre>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.neg_l2_distance","title":"neg_l2_distance","text":"<pre><code>neg_l2_distance(y_true: NDArray[T], y_pred: NDArray[T]) -&gt; NDArray[T]\n</code></pre> <p>Point-wise negative \\(l_2\\) distance between two arrays</p> PARAMETER  DESCRIPTION <code>y_true</code> <p>Array of true values (e.g. labels)</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>y_pred</code> <p>Array of estimated values (e.g. model predictions)</p> <p> TYPE: <code>NDArray[T]</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>Array with point-wise negative \\(l_2\\) distances between labels and model</p> <code>NDArray[T]</code> <p>predictions</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def neg_l2_distance(y_true: NDArray[T], y_pred: NDArray[T]) -&gt; NDArray[T]:\n    r\"\"\"Point-wise negative $l_2$ distance between two arrays\n\n    Args:\n        y_true: Array of true values (e.g. labels)\n        y_pred: Array of estimated values (e.g. model predictions)\n\n    Returns:\n        Array with point-wise negative $l_2$ distances between labels and model\n        predictions\n    \"\"\"\n    return -np.square(np.array(y_pred - y_true), dtype=y_pred.dtype)\n</code></pre>"},{"location":"api/pydvl/value/shapley/","title":"Shapley","text":""},{"location":"api/pydvl/value/shapley/#pydvl.value.shapley","title":"pydvl.value.shapley","text":"<p>This package holds all routines for the computation of Shapley Data value. Users will want to use compute_shapley_values or compute_semivalues as interfaces to most methods defined in the modules.</p> <p>Please refer to the guide on data valuation for an overview of all methods.</p>"},{"location":"api/pydvl/value/shapley/classwise/","title":"Classwise","text":""},{"location":"api/pydvl/value/shapley/classwise/#pydvl.value.shapley.classwise","title":"pydvl.value.shapley.classwise","text":"<p>Class-wise Shapley (Schoch et al., 2022)<sup>1</sup> offers a Shapley framework tailored for classification problems. Let \\(D\\) be a dataset, \\(D_{y_i}\\) be the subset of \\(D\\) with labels \\(y_i\\), and \\(D_{-y_i}\\) be the complement of \\(D_{y_i}\\) in \\(D\\). The key idea is that a sample \\((x_i, y_i)\\), might enhance the overall performance on \\(D\\), while being detrimental for the performance on \\(D_{y_i}\\). The Class-wise value is defined as:</p> \\[ v_u(i) = \\frac{1}{2^{|D_{-y_i}|}} \\sum_{S_{-y_i}} \\frac{1}{|D_{y_i}|!} \\sum_{S_{y_i}} \\binom{|D_{y_i}|-1}{|S_{y_i}|}^{-1} [u( S_{y_i} \\cup \\{i\\} | S_{-y_i} ) \u2212 u( S_{y_i} | S_{-y_i})], \\] <p>where \\(S_{y_i} \\subseteq D_{y_i} \\setminus \\{i\\}\\) and \\(S_{-y_i} \\subseteq D_{-y_i}\\).</p> <p>Analysis of Class-wise Shapley</p> <p>For a detailed analysis of the method, with comparison to other valuation techniques, please refer to the main documentation.</p> <p>In practice, the quantity above is estimated using Monte Carlo sampling of the powerset and the set of index permutations. This results in the estimator</p> \\[ v_u(i) = \\frac{1}{K} \\sum_k \\frac{1}{L} \\sum_l [u(\\sigma^{(l)}_{:i} \\cup \\{i\\} | S^{(k)} ) \u2212 u( \\sigma^{(l)}_{:i} | S^{(k)})], \\] <p>with \\(S^{(1)}, \\dots, S^{(K)} \\subseteq T_{-y_i},\\) \\(\\sigma^{(1)}, \\dots, \\sigma^{(L)} \\in \\Pi(T_{y_i}\\setminus\\{i\\}),\\) and \\(\\sigma^{(l)}_{:i}\\) denoting the set of indices in permutation \\(\\sigma^{(l)}\\) before the position where \\(i\\) appears. The sets \\(T_{y_i}\\) and \\(T_{-y_i}\\) are the training sets for the labels \\(y_i\\) and \\(-y_i\\), respectively.</p> Notes for derivation of test cases <p>The unit tests include the following manually constructed data: Let \\(D=\\{(1,0),(2,0),(3,0),(4,1)\\}\\) be the test set and \\(T=\\{(1,0),(2,0),(3,1),(4,1)\\}\\) the train set. This specific dataset is chosen as it allows to solve the model</p> \\[y = \\max(0, \\min(1, \\text{round}(\\beta^T x)))\\] <p>in closed form \\(\\beta = \\frac{\\text{dot}(x, y)}{\\text{dot}(x, x)}\\). From the closed-form solution, the tables for in-class accuracy \\(a_S(D_{y_i})\\) and out-of-class accuracy  \\(a_S(D_{-y_i})\\) can be calculated. By using these tables and setting  \\(\\{S^{(1)}, \\dots, S^{(K)}\\} = 2^{T_{-y_i}}\\) and  \\(\\{\\sigma^{(1)}, \\dots, \\sigma^{(L)}\\} = \\Pi(T_{y_i}\\setminus\\{i\\})\\), the Monte Carlo estimator can be evaluated (\\(2^M\\) is the powerset of \\(M\\)). The details of the derivation are left to the eager reader.</p>"},{"location":"api/pydvl/value/shapley/classwise/#pydvl.value.shapley.classwise--references","title":"References","text":"<ol> <li> <p>Schoch, Stephanie, Haifeng Xu, and Yangfeng Ji. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification. In Proc. of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). New Orleans, Louisiana, USA, 2022.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/classwise/#pydvl.value.shapley.classwise.ClasswiseScorer","title":"ClasswiseScorer","text":"<pre><code>ClasswiseScorer(\n    scoring: Union[str, ScorerCallable] = \"accuracy\",\n    default: float = 0.0,\n    range: Tuple[float, float] = (0, 1),\n    in_class_discount_fn: Callable[[float], float] = lambda x: x,\n    out_of_class_discount_fn: Callable[[float], float] = np.exp,\n    initial_label: Optional[int] = None,\n    name: Optional[str] = None,\n)\n</code></pre> <p>             Bases: <code>Scorer</code></p> <p>A Scorer designed for evaluation in classification problems. Its value is computed from an in-class and an out-of-class \"inner score\" (Schoch et al., 2022) <sup>1</sup>. Let \\(S\\) be the training set and \\(D\\) be the valuation set. For each label \\(c\\), \\(D\\) is factorized into two disjoint sets: \\(D_c\\) for in-class instances and \\(D_{-c}\\) for out-of-class instances. The score combines an in-class metric of performance, adjusted by a discounted out-of-class metric. These inner scores must be provided upon construction or default to accuracy. They are combined into:</p> \\[ u(S_{y_i}) = f(a_S(D_{y_i}))\\ g(a_S(D_{-y_i})), \\] <p>where \\(f\\) and \\(g\\) are continuous, monotonic functions. For a detailed explanation, refer to section four of (Schoch et al., 2022)<sup> 1</sup>.</p> <p>Warning</p> <p>Metrics must support multiple class labels if you intend to apply them to a multi-class problem. For instance, the metric 'accuracy' supports multiple classes, but the metric <code>f1</code> does not. For a two-class classification problem, using <code>f1_weighted</code> is essentially equivalent to using <code>accuracy</code>.</p> PARAMETER  DESCRIPTION <code>scoring</code> <p>Name of the scoring function or a callable that can be passed to Scorer.</p> <p> TYPE: <code>Union[str, ScorerCallable]</code> DEFAULT: <code>'accuracy'</code> </p> <code>default</code> <p>Score to use when a model fails to provide a number, e.g. when too little was used to train it, or errors arise.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>range</code> <p>Numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation. If not provided, it can be read from the <code>scoring</code> object if it provides it, for instance if it was constructed with compose_score.</p> <p> TYPE: <code>Tuple[float, float]</code> DEFAULT: <code>(0, 1)</code> </p> <code>in_class_discount_fn</code> <p>Continuous, monotonic increasing function used to discount the in-class score.</p> <p> TYPE: <code>Callable[[float], float]</code> DEFAULT: <code>lambda x: x</code> </p> <code>out_of_class_discount_fn</code> <p>Continuous, monotonic increasing function used to discount the out-of-class score.</p> <p> TYPE: <code>Callable[[float], float]</code> DEFAULT: <code>exp</code> </p> <code>initial_label</code> <p>Set initial label (for the first iteration)</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>name</code> <p>Name of the scorer. If not provided, the name of the inner scoring function will be prefixed by <code>classwise</code>.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <p>New in version 0.7.1</p> Source code in <code>src/pydvl/value/shapley/classwise.py</code> <pre><code>def __init__(\n    self,\n    scoring: Union[str, ScorerCallable] = \"accuracy\",\n    default: float = 0.0,\n    range: Tuple[float, float] = (0, 1),\n    in_class_discount_fn: Callable[[float], float] = lambda x: x,\n    out_of_class_discount_fn: Callable[[float], float] = np.exp,\n    initial_label: Optional[int] = None,\n    name: Optional[str] = None,\n):\n    disc_score_in_class = in_class_discount_fn(range[1])\n    disc_score_out_of_class = out_of_class_discount_fn(range[1])\n    transformed_range = (0, disc_score_in_class * disc_score_out_of_class)\n    super().__init__(\n        scoring=scoring,\n        range=transformed_range,\n        default=default,\n        name=name or f\"classwise {str(scoring)}\",\n    )\n    self._in_class_discount_fn = in_class_discount_fn\n    self._out_of_class_discount_fn = out_of_class_discount_fn\n    self.label = initial_label\n</code></pre>"},{"location":"api/pydvl/value/shapley/classwise/#pydvl.value.shapley.classwise.ClasswiseScorer.estimate_in_class_and_out_of_class_score","title":"estimate_in_class_and_out_of_class_score","text":"<pre><code>estimate_in_class_and_out_of_class_score(\n    model: SupervisedModel,\n    x_test: NDArray[float_],\n    y_test: NDArray[int_],\n    rescale_scores: bool = True,\n) -&gt; Tuple[float, float]\n</code></pre> <p>Computes in-class and out-of-class scores using the provided inner scoring function. The result is</p> \\[ a_S(D=\\{(x_1, y_1), \\dots, (x_K, y_K)\\}) = \\frac{1}{N} \\sum_k s(y(x_k), y_k). \\] <p>In this context, for label \\(c\\) calculations are executed twice: once for \\(D_c\\) and once for \\(D_{-c}\\) to determine the in-class and out-of-class scores, respectively. By default, the raw scores are multiplied by \\(\\frac{|D_c|}{|D|}\\) and \\(\\frac{|D_{-c}|}{|D|}\\), respectively. This is done to ensure that both scores are of the same order of magnitude. This normalization is particularly useful when the inner score function \\(a_S\\) is calculated by an estimator of the form \\(\\frac{1}{N} \\sum_i x_i\\), e.g. the accuracy.</p> PARAMETER  DESCRIPTION <code>model</code> <p>Model used for computing the score on the validation set.</p> <p> TYPE: <code>SupervisedModel</code> </p> <code>x_test</code> <p>Array containing the features of the classification problem.</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>y_test</code> <p>Array containing the labels of the classification problem.</p> <p> TYPE: <code>NDArray[int_]</code> </p> <code>rescale_scores</code> <p>If set to True, the scores will be denormalized. This is particularly useful when the inner score function \\(a_S\\) is calculated by an estimator of the form \\(\\frac{1}{N} \\sum_i x_i\\).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>Tuple[float, float]</code> <p>Tuple containing the in-class and out-of-class scores.</p> Source code in <code>src/pydvl/value/shapley/classwise.py</code> <pre><code>def estimate_in_class_and_out_of_class_score(\n    self,\n    model: SupervisedModel,\n    x_test: NDArray[np.float_],\n    y_test: NDArray[np.int_],\n    rescale_scores: bool = True,\n) -&gt; Tuple[float, float]:\n    r\"\"\"\n    Computes in-class and out-of-class scores using the provided inner\n    scoring function. The result is\n\n    $$\n    a_S(D=\\{(x_1, y_1), \\dots, (x_K, y_K)\\}) = \\frac{1}{N} \\sum_k s(y(x_k), y_k).\n    $$\n\n    In this context, for label $c$ calculations are executed twice: once for $D_c$\n    and once for $D_{-c}$ to determine the in-class and out-of-class scores,\n    respectively. By default, the raw scores are multiplied by $\\frac{|D_c|}{|D|}$\n    and $\\frac{|D_{-c}|}{|D|}$, respectively. This is done to ensure that both\n    scores are of the same order of magnitude. This normalization is particularly\n    useful when the inner score function $a_S$ is calculated by an estimator of the\n    form $\\frac{1}{N} \\sum_i x_i$, e.g. the accuracy.\n\n    Args:\n        model: Model used for computing the score on the validation set.\n        x_test: Array containing the features of the classification problem.\n        y_test: Array containing the labels of the classification problem.\n        rescale_scores: If set to True, the scores will be denormalized. This is\n            particularly useful when the inner score function $a_S$ is calculated by\n            an estimator of the form $\\frac{1}{N} \\sum_i x_i$.\n\n    Returns:\n        Tuple containing the in-class and out-of-class scores.\n    \"\"\"\n    scorer = self._scorer\n    label_set_match = y_test == self.label\n    label_set = np.where(label_set_match)[0]\n    num_classes = len(np.unique(y_test))\n\n    if len(label_set) == 0:\n        return 0, 1 / (num_classes - 1)\n\n    complement_label_set = np.where(~label_set_match)[0]\n    in_class_score = scorer(model, x_test[label_set], y_test[label_set])\n    out_of_class_score = scorer(\n        model, x_test[complement_label_set], y_test[complement_label_set]\n    )\n\n    if rescale_scores:\n        n_in_class = np.count_nonzero(y_test == self.label)\n        n_out_of_class = len(y_test) - n_in_class\n        in_class_score *= n_in_class / (n_in_class + n_out_of_class)\n        out_of_class_score *= n_out_of_class / (n_in_class + n_out_of_class)\n\n    return in_class_score, out_of_class_score\n</code></pre>"},{"location":"api/pydvl/value/shapley/classwise/#pydvl.value.shapley.classwise.compute_classwise_shapley_values","title":"compute_classwise_shapley_values","text":"<pre><code>compute_classwise_shapley_values(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    truncation: TruncationPolicy,\n    done_sample_complements: Optional[StoppingCriterion] = None,\n    normalize_values: bool = True,\n    use_default_scorer_value: bool = True,\n    min_elements_per_label: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes an approximate Class-wise Shapley value by sampling independent permutations of the index set for each label and index sets sampled from the powerset of the complement (with respect to the currently evaluated label), approximating the sum:</p> \\[ v_u(i) = \\frac{1}{K} \\sum_k \\frac{1}{L} \\sum_l [u(\\sigma^{(l)}_{:i} \\cup \\{i\\} | S^{(k)} ) \u2212 u( \\sigma^{(l)}_{:i} | S^{(k)})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears and \\(S\\) is a subset of the index set of all other labels (see the main documentation for details).</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object containing model, data, and scoring function. The scorer must be of type ClasswiseScorer.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Function that checks whether the computation needs to stop.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>truncation</code> <p>Callable function that decides whether to interrupt processing a permutation and set subsequent marginals to zero.</p> <p> TYPE: <code>TruncationPolicy</code> </p> <code>done_sample_complements</code> <p>Function checking whether computation needs to stop. Otherwise, it will resample conditional sets until the stopping criterion is met.</p> <p> TYPE: <code>Optional[StoppingCriterion]</code> DEFAULT: <code>None</code> </p> <code>normalize_values</code> <p>Indicates whether to normalize the values by the variation in each class times their in-class accuracy.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>done_sample_complements</code> <p>Number of times to resample the complement set for each permutation.</p> <p> TYPE: <code>Optional[StoppingCriterion]</code> DEFAULT: <code>None</code> </p> <code>use_default_scorer_value</code> <p>The first set of indices is the sampled complement set. Unless not otherwise specified, the default scorer value is used for this. If it is set to false, the base score is calculated from the utility.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>min_elements_per_label</code> <p>The minimum number of elements for each opposite label.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to run.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>ValuationResult object containing computed data values.</p> <p>New in version 0.7.1</p> Source code in <code>src/pydvl/value/shapley/classwise.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef compute_classwise_shapley_values(\n    u: Utility,\n    *,\n    done: StoppingCriterion,\n    truncation: TruncationPolicy,\n    done_sample_complements: Optional[StoppingCriterion] = None,\n    normalize_values: bool = True,\n    use_default_scorer_value: bool = True,\n    min_elements_per_label: int = 1,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    r\"\"\"\n    Computes an approximate Class-wise Shapley value by sampling independent\n    permutations of the index set for each label and index sets sampled from the\n    powerset of the complement (with respect to the currently evaluated label),\n    approximating the sum:\n\n    $$\n    v_u(i) = \\frac{1}{K} \\sum_k \\frac{1}{L} \\sum_l\n    [u(\\sigma^{(l)}_{:i} \\cup \\{i\\} | S^{(k)} ) \u2212 u( \\sigma^{(l)}_{:i} | S^{(k)})],\n    $$\n\n    where $\\sigma_{:i}$ denotes the set of indices in permutation sigma before\n    the position where $i$ appears and $S$ is a subset of the index set of all\n    other labels (see [the main documentation][class-wise-shapley] for\n    details).\n\n    Args:\n        u: Utility object containing model, data, and scoring function. The\n            scorer must be of type\n            [ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer].\n        done: Function that checks whether the computation needs to stop.\n        truncation: Callable function that decides whether to interrupt processing a\n            permutation and set subsequent marginals to zero.\n        done_sample_complements: Function checking whether computation needs to stop.\n            Otherwise, it will resample conditional sets until the stopping criterion is\n            met.\n        normalize_values: Indicates whether to normalize the values by the variation\n            in each class times their in-class accuracy.\n        done_sample_complements: Number of times to resample the complement set\n            for each permutation.\n        use_default_scorer_value: The first set of indices is the sampled complement\n            set. Unless not otherwise specified, the default scorer value is used for\n            this. If it is set to false, the base score is calculated from the utility.\n        min_elements_per_label: The minimum number of elements for each opposite\n            label.\n        n_jobs: Number of parallel jobs to run.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        ValuationResult object containing computed data values.\n\n    !!! tip \"New in version 0.7.1\"\n    \"\"\"\n    dim_correct = u.data.y_train.ndim == 1 and u.data.y_test.ndim == 1\n    is_integral = all(\n        map(\n            lambda v: isinstance(v, numbers.Integral), (*u.data.y_train, *u.data.y_test)\n        )\n    )\n    if not dim_correct or not is_integral:\n        raise ValueError(\n            \"The supplied dataset has to be a 1-dimensional classification dataset.\"\n        )\n\n    if not isinstance(u.scorer, ClasswiseScorer):\n        raise ValueError(\n            \"Please set a subclass of ClasswiseScorer object as scorer object of the\"\n            \" utility. See scoring argument of Utility.\"\n        )\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n    u_ref = parallel_backend.put(u)\n    n_jobs = parallel_backend.effective_n_jobs(n_jobs)\n    n_submitted_jobs = 2 * n_jobs\n\n    pbar = tqdm(disable=not progress, position=0, total=100, unit=\"%\")\n    algorithm = \"classwise_shapley\"\n    accumulated_result = ValuationResult.zeros(\n        algorithm=algorithm, indices=u.data.indices, data_names=u.data.data_names\n    )\n    terminate_exec = False\n    seed_sequence = ensure_seed_sequence(seed)\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    with parallel_backend.executor(max_workers=n_jobs) as executor:\n        pending: Set[Future] = set()\n        while True:\n            completed_futures, pending = wait(\n                pending, timeout=60, return_when=FIRST_COMPLETED\n            )\n            for future in completed_futures:\n                accumulated_result += future.result()\n                if done(accumulated_result):\n                    terminate_exec = True\n                    break\n\n            pbar.n = 100 * done.completion()\n            pbar.refresh()\n            if terminate_exec:\n                break\n\n            n_remaining_slots = n_submitted_jobs - len(pending)\n            seeds = seed_sequence.spawn(n_remaining_slots)\n            for i in range(n_remaining_slots):\n                future = executor.submit(\n                    _permutation_montecarlo_classwise_shapley_one_step,\n                    u_ref,\n                    truncation=truncation,\n                    done_sample_complements=done_sample_complements,\n                    use_default_scorer_value=use_default_scorer_value,\n                    min_elements_per_label=min_elements_per_label,\n                    algorithm_name=algorithm,\n                    seed=seeds[i],\n                )\n                pending.add(future)\n\n    result = accumulated_result\n    if normalize_values:\n        result = _normalize_classwise_shapley_values(result, u)\n\n    return result\n</code></pre>"},{"location":"api/pydvl/value/shapley/common/","title":"Common","text":""},{"location":"api/pydvl/value/shapley/common/#pydvl.value.shapley.common","title":"pydvl.value.shapley.common","text":""},{"location":"api/pydvl/value/shapley/common/#pydvl.value.shapley.common.compute_shapley_values","title":"compute_shapley_values","text":"<pre><code>compute_shapley_values(\n    u: Utility,\n    *,\n    done: StoppingCriterion = MaxChecks(None),\n    mode: ShapleyMode = ShapleyMode.TruncatedMontecarlo,\n    n_jobs: int = 1,\n    seed: Optional[Seed] = None,\n    **kwargs\n) -&gt; ValuationResult\n</code></pre> <p>Umbrella method to compute Shapley values with any of the available algorithms.</p> <p>See Data valuation for an overview.</p> <p>The following algorithms are available. Note that the exact methods can only work with very small datasets and are thus intended only for testing. Some algorithms also accept additional arguments, please refer to the documentation of each particular method.</p> <ul> <li><code>combinatorial_exact</code>: uses the combinatorial implementation of data   Shapley. Implemented in   combinatorial_exact_shapley().</li> <li><code>combinatorial_montecarlo</code>:  uses the approximate Monte Carlo   implementation of combinatorial data Shapley. Implemented in   combinatorial_montecarlo_shapley().</li> <li><code>permutation_exact</code>: uses the permutation-based implementation of data   Shapley. Computation is not parallelized. Implemented in   permutation_exact_shapley().</li> <li><code>permutation_montecarlo</code>: uses the approximate Monte Carlo   implementation of permutation data Shapley. Accepts a   TruncationPolicy to stop   computing marginals. Implemented in   permutation_montecarlo_shapley().</li> <li><code>owen_sampling</code>: Uses the Owen continuous extension of the utility   function to the unit cube. Implemented in   owen_sampling_shapley(). This   method does not take a StoppingCriterion   but instead requires a parameter <code>q_max</code> for the number of subdivisions   of the unit interval to use for integration, and another parameter   <code>n_samples</code> for the number of subsets to sample for each \\(q\\).</li> <li><code>owen_halved</code>: Same as 'owen_sampling' but uses correlated samples in the   expectation. Implemented in   owen_sampling_shapley().   This method  requires an additional parameter <code>q_max</code> for the number of   subdivisions of the interval [0,0.5] to use for integration, and another   parameter <code>n_samples</code> for the number of subsets to sample for each \\(q\\).</li> <li><code>group_testing</code>: estimates differences of Shapley values and solves a   constraint satisfaction problem. High sample complexity, not recommended.   Implemented in group_testing_shapley(). This   method does not take a StoppingCriterion   but instead requires a parameter <code>n_samples</code> for the number of   iterations to run.</li> </ul> <p>Additionally, one can use model-specific methods:</p> <ul> <li><code>knn</code>: Exact method for K-Nearest neighbour models. Implemented in   knn_shapley().</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Object used to determine when to stop the computation for Monte Carlo methods. The default is to stop after 100 iterations. See the available criteria in stopping. It is possible to combine several of them using boolean operators. Some methods ignore this argument, others require specific subtypes.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxChecks(None)</code> </p> <code>n_jobs</code> <p>Number of parallel jobs (available only to some methods)</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>Choose which shapley algorithm to use. See ShapleyMode for a list of allowed value.</p> <p> TYPE: <code>ShapleyMode</code> DEFAULT: <code>TruncatedMontecarlo</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/shapley/common.py</code> <pre><code>def compute_shapley_values(\n    u: Utility,\n    *,\n    done: StoppingCriterion = MaxChecks(None),\n    mode: ShapleyMode = ShapleyMode.TruncatedMontecarlo,\n    n_jobs: int = 1,\n    seed: Optional[Seed] = None,\n    **kwargs,\n) -&gt; ValuationResult:\n    \"\"\"Umbrella method to compute Shapley values with any of the available\n    algorithms.\n\n    See [Data valuation][data-valuation] for an overview.\n\n    The following algorithms are available. Note that the exact methods can only\n    work with very small datasets and are thus intended only for testing. Some\n    algorithms also accept additional arguments, please refer to the\n    documentation of each particular method.\n\n    - `combinatorial_exact`: uses the combinatorial implementation of data\n      Shapley. Implemented in\n      [combinatorial_exact_shapley()][pydvl.value.shapley.naive.combinatorial_exact_shapley].\n    - `combinatorial_montecarlo`:  uses the approximate Monte Carlo\n      implementation of combinatorial data Shapley. Implemented in\n      [combinatorial_montecarlo_shapley()][pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley].\n    - `permutation_exact`: uses the permutation-based implementation of data\n      Shapley. Computation is **not parallelized**. Implemented in\n      [permutation_exact_shapley()][pydvl.value.shapley.naive.permutation_exact_shapley].\n    - `permutation_montecarlo`: uses the approximate Monte Carlo\n      implementation of permutation data Shapley. Accepts a\n      [TruncationPolicy][pydvl.value.shapley.truncated.TruncationPolicy] to stop\n      computing marginals. Implemented in\n      [permutation_montecarlo_shapley()][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].\n    - `owen_sampling`: Uses the Owen continuous extension of the utility\n      function to the unit cube. Implemented in\n      [owen_sampling_shapley()][pydvl.value.shapley.owen.owen_sampling_shapley]. This\n      method does not take a [StoppingCriterion][pydvl.value.stopping.StoppingCriterion]\n      but instead requires a parameter `q_max` for the number of subdivisions\n      of the unit interval to use for integration, and another parameter\n      `n_samples` for the number of subsets to sample for each $q$.\n    - `owen_halved`: Same as 'owen_sampling' but uses correlated samples in the\n      expectation. Implemented in\n      [owen_sampling_shapley()][pydvl.value.shapley.owen.owen_sampling_shapley].\n      This method  requires an additional parameter `q_max` for the number of\n      subdivisions of the interval [0,0.5] to use for integration, and another\n      parameter `n_samples` for the number of subsets to sample for each $q$.\n    - `group_testing`: estimates differences of Shapley values and solves a\n      constraint satisfaction problem. High sample complexity, not recommended.\n      Implemented in [group_testing_shapley()][pydvl.value.shapley.gt.group_testing_shapley]. This\n      method does not take a [StoppingCriterion][pydvl.value.stopping.StoppingCriterion]\n      but instead requires a parameter `n_samples` for the number of\n      iterations to run.\n\n    Additionally, one can use model-specific methods:\n\n    - `knn`: Exact method for K-Nearest neighbour models. Implemented in\n      [knn_shapley()][pydvl.value.shapley.knn.knn_shapley].\n\n    Args:\n        u: [Utility][pydvl.utils.utility.Utility] object with model, data, and\n            scoring function.\n        done: Object used to determine when to stop the computation for Monte\n            Carlo methods. The default is to stop after 100 iterations. See the\n            available criteria in [stopping][pydvl.value.stopping]. It is\n            possible to combine several of them using boolean operators. Some\n            methods ignore this argument, others require specific subtypes.\n        n_jobs: Number of parallel jobs (available only to some methods)\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        mode: Choose which shapley algorithm to use. See\n            [ShapleyMode][pydvl.value.shapley.ShapleyMode] for a list of allowed\n            value.\n\n    Returns:\n        Object with the results.\n\n    \"\"\"\n    progress: bool = kwargs.pop(\"progress\", False)\n\n    if mode not in list(ShapleyMode):\n        raise ValueError(f\"Invalid value encountered in {mode=}\")\n\n    if mode in (\n        ShapleyMode.PermutationMontecarlo,\n        ShapleyMode.ApproShapley,\n        ShapleyMode.TruncatedMontecarlo,\n    ):\n        truncation = kwargs.pop(\"truncation\", NoTruncation())\n        return permutation_montecarlo_shapley(  # type: ignore\n            u=u,\n            done=done,\n            truncation=truncation,\n            n_jobs=n_jobs,\n            seed=seed,\n            progress=progress,\n            **kwargs,\n        )\n    elif mode == ShapleyMode.CombinatorialMontecarlo:\n        return combinatorial_montecarlo_shapley(  # type: ignore\n            u, done=done, n_jobs=n_jobs, seed=seed, progress=progress\n        )\n    elif mode == ShapleyMode.CombinatorialExact:\n        return combinatorial_exact_shapley(u, n_jobs=n_jobs, progress=progress)  # type: ignore\n    elif mode == ShapleyMode.PermutationExact:\n        return permutation_exact_shapley(u, progress=progress)\n    elif mode == ShapleyMode.Owen or mode == ShapleyMode.OwenAntithetic:\n        if kwargs.get(\"n_samples\") is None:\n            raise ValueError(\"n_samples cannot be None for Owen methods\")\n        if kwargs.get(\"max_q\") is None:\n            raise ValueError(\"Owen Sampling requires max_q for the outer integral\")\n\n        method = (\n            OwenAlgorithm.Standard\n            if mode == ShapleyMode.Owen\n            else OwenAlgorithm.Antithetic\n        )\n        return owen_sampling_shapley(  # type: ignore\n            u,\n            n_samples=int(kwargs.get(\"n_samples\", -1)),\n            max_q=int(kwargs.get(\"max_q\", -1)),\n            method=method,\n            n_jobs=n_jobs,\n            seed=seed,\n        )\n    elif mode == ShapleyMode.KNN:\n        return knn_shapley(u, progress=progress)\n    elif mode == ShapleyMode.GroupTesting:\n        n_samples = kwargs.pop(\"n_samples\")\n        if n_samples is None:\n            raise ValueError(\"n_samples cannot be None for Group Testing\")\n        epsilon = kwargs.pop(\"epsilon\")\n        if epsilon is None:\n            raise ValueError(\"Group Testing requires error bound epsilon\")\n        delta = kwargs.pop(\"delta\", 0.05)\n        return group_testing_shapley(  # type: ignore\n            u,\n            epsilon=float(epsilon),\n            delta=delta,\n            n_samples=int(n_samples),\n            n_jobs=n_jobs,\n            progress=progress,\n            seed=seed,\n            **kwargs,\n        )\n    else:\n        raise ValueError(f\"Invalid value encountered in {mode=}\")\n</code></pre>"},{"location":"api/pydvl/value/shapley/gt/","title":"Gt","text":""},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt","title":"pydvl.value.shapley.gt","text":"<p>This module implements Group Testing for the approximation of Shapley values, as introduced in (Jia, R. et al., 2019)<sup>1</sup>. The sampling of index subsets is done in such a way that an approximation to the true Shapley values can be computed with guarantees.</p> <p>Warning</p> <p>This method is very inefficient. Potential improvements to the implementation notwithstanding, convergence seems to be very slow (in terms of evaluations of the utility required). We recommend other Monte Carlo methods instead.</p> <p>You can read more in the documentation.</p> <p>New in version 0.4.0</p>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt--references","title":"References","text":"<ol> <li> <p>Jia, R. et al., 2019. Towards Efficient Data Valuation Based on the Shapley Value. In: Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics, pp. 1167\u20131176. PMLR.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt.num_samples_eps_delta","title":"num_samples_eps_delta","text":"<pre><code>num_samples_eps_delta(\n    eps: float, delta: float, n: int, utility_range: float\n) -&gt; int\n</code></pre> <p>Implements the formula in Theorem 3 of (Jia, R. et al., 2019)<sup>1</sup> which gives a lower bound on the number of samples required to obtain an (\u03b5/\u221an,\u03b4/(N(N-1))-approximation to all pair-wise differences of Shapley values, wrt. \\(\\ell_2\\) norm.</p> PARAMETER  DESCRIPTION <code>eps</code> <p>\u03b5</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>\u03b4</p> <p> TYPE: <code>float</code> </p> <code>n</code> <p>Number of data points</p> <p> TYPE: <code>int</code> </p> <code>utility_range</code> <p>Range of the Utility function</p> <p> TYPE: <code>float</code> </p> <p>Returns:     Number of samples from \\(2^{[n]}\\) guaranteeing \u03b5/\u221an-correct Shapley         pair-wise differences of values with probability 1-\u03b4/(N(N-1)).</p> <p>New in version 0.4.0</p> Source code in <code>src/pydvl/value/shapley/gt.py</code> <pre><code>def num_samples_eps_delta(\n    eps: float, delta: float, n: int, utility_range: float\n) -&gt; int:\n    r\"\"\"Implements the formula in Theorem 3 of (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n    which gives a lower bound on the number of samples required to obtain an\n    (\u03b5/\u221an,\u03b4/(N(N-1))-approximation to all pair-wise differences of Shapley\n    values, wrt. $\\ell_2$ norm.\n\n    Args:\n        eps: \u03b5\n        delta: \u03b4\n        n: Number of data points\n        utility_range: Range of the [Utility][pydvl.utils.utility.Utility] function\n    Returns:\n        Number of samples from $2^{[n]}$ guaranteeing \u03b5/\u221an-correct Shapley\n            pair-wise differences of values with probability 1-\u03b4/(N(N-1)).\n\n    !!! tip \"New in version 0.4.0\"\n\n    \"\"\"\n    constants = _constants(n=n, epsilon=eps, delta=delta, utility_range=utility_range)\n    return int(constants.T)\n</code></pre>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt.group_testing_shapley","title":"group_testing_shapley","text":"<pre><code>group_testing_shapley(\n    u: Utility,\n    n_samples: int,\n    epsilon: float,\n    delta: float,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n    **options: dict\n) -&gt; ValuationResult\n</code></pre> <p>Implements group testing for approximation of Shapley values as described in (Jia, R. et al., 2019)<sup>1</sup>.</p> <p>Warning</p> <p>This method is very inefficient. It requires several orders of magnitude more evaluations of the utility than others in montecarlo. It also uses several intermediate objects like the results from the runners and the constraint matrices which can become rather large.</p> <p>By picking a specific distribution over subsets, the differences in Shapley values can be approximated with a Monte Carlo sum. These are then used to solve for the individual values in a feasibility problem.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Number of tests to perform. Use num_samples_eps_delta to estimate this.</p> <p> TYPE: <code>int</code> </p> <code>epsilon</code> <p>From the (\u03b5,\u03b4) sample bound. Use the same as for the estimation of <code>n_iterations</code>.</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>From the (\u03b5,\u03b4) sample bound. Use the same as for the estimation of <code>n_iterations</code>.</p> <p> TYPE: <code>float</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use. Each worker performs a chunk of all tests (i.e. utility evaluations).</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>options</code> <p>Additional options to pass to cvxpy.Problem.solve(). E.g. to change the solver (which defaults to <code>cvxpy.SCS</code>) pass <code>solver=cvxpy.CVXOPT</code>.</p> <p> TYPE: <code>dict</code> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.5.0</p> <p>Changed the solver to cvxpy instead of scipy's linprog. Added the ability to pass arbitrary options to it.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/shapley/gt.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef group_testing_shapley(\n    u: Utility,\n    n_samples: int,\n    epsilon: float,\n    delta: float,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n    **options: dict,\n) -&gt; ValuationResult:\n    \"\"\"Implements group testing for approximation of Shapley values as described\n    in (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n\n    !!! Warning\n        This method is very inefficient. It requires several orders of magnitude\n        more evaluations of the utility than others in\n        [montecarlo][pydvl.value.shapley.montecarlo]. It also uses several intermediate\n        objects like the results from the runners and the constraint matrices\n        which can become rather large.\n\n    By picking a specific distribution over subsets, the differences in Shapley\n    values can be approximated with a Monte Carlo sum. These are then used to\n    solve for the individual values in a feasibility problem.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_samples: Number of tests to perform. Use\n            [num_samples_eps_delta][pydvl.value.shapley.gt.num_samples_eps_delta]\n            to estimate this.\n        epsilon: From the (\u03b5,\u03b4) sample bound. Use the same as for the\n            estimation of `n_iterations`.\n        delta: From the (\u03b5,\u03b4) sample bound. Use the same as for the\n            estimation of `n_iterations`.\n        n_jobs: Number of parallel jobs to use. Each worker performs a chunk\n            of all tests (i.e. utility evaluations).\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        options: Additional options to pass to\n            [cvxpy.Problem.solve()](https://www.cvxpy.org/tutorial/advanced/index.html#solve-method-options).\n            E.g. to change the solver (which defaults to `cvxpy.SCS`) pass\n            `solver=cvxpy.CVXOPT`.\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"New in version 0.4.0\"\n\n    !!! tip \"Changed in version 0.5.0\"\n        Changed the solver to cvxpy instead of scipy's linprog. Added the ability\n        to pass arbitrary options to it.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n\n    n = len(u.data.indices)\n\n    const = _constants(\n        n=n,\n        epsilon=epsilon,\n        delta=delta,\n        utility_range=u.score_range.max() - u.score_range.min(),\n    )\n    T = n_samples\n    if T &lt; const.T:\n        log.warning(\n            f\"n_samples of {T} are below the required {const.T} for the \"\n            f\"\u03b5={epsilon:.02f} guarantee at \u03b4={1 - delta:.02f} probability\"\n        )\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    samples_per_job = max(1, n_samples // parallel_backend.effective_n_jobs(n_jobs))\n\n    def reducer(\n        results_it: Iterable[Tuple[NDArray, NDArray]]\n    ) -&gt; Tuple[NDArray, NDArray]:\n        return np.concatenate(list(x[0] for x in results_it)).astype(\n            np.float_\n        ), np.concatenate(list(x[1] for x in results_it)).astype(np.int_)\n\n    seed_sequence = ensure_seed_sequence(seed)\n    map_reduce_seed_sequence, cvxpy_seed = tuple(seed_sequence.spawn(2))\n\n    map_reduce_job: MapReduceJob[Utility, Tuple[NDArray, NDArray]] = MapReduceJob(\n        u,\n        map_func=_group_testing_shapley,\n        reduce_func=reducer,\n        map_kwargs=dict(n_samples=samples_per_job, progress=progress),\n        parallel_backend=parallel_backend,\n        n_jobs=n_jobs,\n    )\n    uu, betas = map_reduce_job(seed=map_reduce_seed_sequence)\n\n    # Matrix of estimated differences. See Eqs. (3) and (4) in the paper.\n    C = np.zeros(shape=(n, n))\n    for i in range(n):\n        for j in range(i + 1, n):\n            C[i, j] = np.dot(uu, betas[:, i] - betas[:, j])\n    C *= const.Z / T\n    total_utility = u(u.data.indices)\n\n    ###########################################################################\n    # Solution of the constraint problem with cvxpy\n\n    v = cp.Variable(n)\n    constraints = [cp.sum(v) == total_utility]\n    for i in range(n):\n        for j in range(i + 1, n):\n            constraints.append(v[i] - v[j] &lt;= epsilon + C[i, j])\n            constraints.append(v[j] - v[i] &lt;= epsilon - C[i, j])\n\n    problem = cp.Problem(cp.Minimize(0), constraints)\n    solver = options.pop(\"solver\", cp.SCS)\n    problem.solve(solver=solver, **options)\n\n    if problem.status != \"optimal\":\n        log.warning(f\"cvxpy returned status {problem.status}\")\n        values = (\n            np.nan * np.ones_like(u.data.indices)\n            if not hasattr(v.value, \"__len__\")\n            else v.value\n        )\n        status = Status.Failed\n    else:\n        values = v.value\n        status = Status.Converged\n\n    return ValuationResult(\n        algorithm=\"group_testing_shapley\",\n        status=status,\n        values=values,\n        data_names=u.data.data_names,\n        solver_status=problem.status,\n    )\n</code></pre>"},{"location":"api/pydvl/value/shapley/knn/","title":"Knn","text":""},{"location":"api/pydvl/value/shapley/knn/#pydvl.value.shapley.knn","title":"pydvl.value.shapley.knn","text":"<p>This module contains Shapley computations for K-Nearest Neighbours.</p> <p>Todo</p> <p>Implement approximate KNN computation for sublinear complexity</p>"},{"location":"api/pydvl/value/shapley/knn/#pydvl.value.shapley.knn--references","title":"References","text":"<ol> <li> <p>Jia, R. et al., 2019. Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms. In: Proceedings of the VLDB Endowment, Vol. 12, No. 11, pp. 1610\u20131623.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/knn/#pydvl.value.shapley.knn.knn_shapley","title":"knn_shapley","text":"<pre><code>knn_shapley(u: Utility, *, progress: bool = True) -&gt; ValuationResult\n</code></pre> <p>Computes exact Shapley values for a KNN classifier.</p> <p>This implements the method described in (Jia, R. et al., 2019)<sup>1</sup>. It exploits the local structure of K-Nearest Neighbours to reduce the number of calls to the utility function to a constant number per index, thus reducing computation time to \\(O(n)\\).</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility with a KNN model to extract parameters from. The object will not be modified nor used other than to call get_params()</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> RAISES DESCRIPTION <code>TypeError</code> <p>If the model in the utility is not a sklearn.neighbors.KNeighborsClassifier.</p> <p>New in version 0.1.0</p> Source code in <code>src/pydvl/value/shapley/knn.py</code> <pre><code>def knn_shapley(u: Utility, *, progress: bool = True) -&gt; ValuationResult:\n    \"\"\"Computes exact Shapley values for a KNN classifier.\n\n    This implements the method described in (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019a\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    It exploits the local structure of K-Nearest Neighbours to reduce the number\n    of calls to the utility function to a constant number per index, thus\n    reducing computation time to $O(n)$.\n\n    Args:\n        u: Utility with a KNN model to extract parameters from. The object\n            will not be modified nor used other than to call [get_params()](\n            &lt;https://scikit-learn.org/stable/modules/generated/sklearn.base.BaseEstimator.html#sklearn.base.BaseEstimator.get_params&gt;)\n        progress: Whether to display a progress bar.\n\n    Returns:\n        Object with the data values.\n\n    Raises:\n        TypeError: If the model in the utility is not a\n            [sklearn.neighbors.KNeighborsClassifier][].\n\n    !!! tip \"New in version 0.1.0\"\n\n    \"\"\"\n    if not isinstance(u.model, KNeighborsClassifier):\n        raise TypeError(\"KNN Shapley requires a K-Nearest Neighbours model\")\n\n    defaults: Dict[str, Union[int, str]] = {\n        \"algorithm\": \"ball_tree\" if u.data.dim &gt;= 20 else \"kd_tree\",\n        \"metric\": \"minkowski\",\n        \"p\": 2,\n    }\n    defaults.update(u.model.get_params())\n    # HACK: NearestNeighbors doesn't support this. There will be more...\n    del defaults[\"weights\"]\n    n_neighbors: int = int(defaults[\"n_neighbors\"])\n    defaults[\"n_neighbors\"] = len(u.data)  # We want all training points sorted\n\n    assert n_neighbors &lt; len(u.data)\n    # assert data.target_dim == 1\n\n    nns = NearestNeighbors(**defaults).fit(u.data.x_train)\n    # closest to farthest\n    _, indices = nns.kneighbors(u.data.x_test)\n\n    values: NDArray[np.float_] = np.zeros_like(u.data.indices, dtype=np.float_)\n    n = len(u.data)\n    yt = u.data.y_train\n    iterator = enumerate(zip(u.data.y_test, indices), start=1)\n    for j, (y, ii) in tqdm(iterator, disable=not progress):\n        value_at_x = int(yt[ii[-1]] == y) / n\n        values[ii[-1]] += (value_at_x - values[ii[-1]]) / j\n        for i in range(n - 2, n_neighbors, -1):  # farthest to closest\n            value_at_x = (\n                values[ii[i + 1]] + (int(yt[ii[i]] == y) - int(yt[ii[i + 1]] == y)) / i\n            )\n            values[ii[i]] += (value_at_x - values[ii[i]]) / j\n        for i in range(n_neighbors, -1, -1):  # farthest to closest\n            value_at_x = (\n                values[ii[i + 1]]\n                + (int(yt[ii[i]] == y) - int(yt[ii[i + 1]] == y)) / n_neighbors\n            )\n            values[ii[i]] += (value_at_x - values[ii[i]]) / j\n\n    return ValuationResult(\n        algorithm=\"knn_shapley\",\n        status=Status.Converged,\n        values=values,\n        data_names=u.data.data_names,\n    )\n</code></pre>"},{"location":"api/pydvl/value/shapley/montecarlo/","title":"Montecarlo","text":""},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo","title":"pydvl.value.shapley.montecarlo","text":"<p>Monte Carlo approximations to Shapley Data values.</p> <p>Warning</p> <p>You probably want to use the common interface provided by compute_shapley_values() instead of directly using the functions in this module.</p> <p>Because exact computation of Shapley values requires \\(\\mathcal{O}(2^n)\\) re-trainings of the model, several Monte Carlo approximations are available. The first two sample from the powerset of the training data directly: combinatorial_montecarlo_shapley() and owen_sampling_shapley(). The latter uses a reformulation in terms of a continuous extension of the utility.</p> <p>Alternatively, employing another reformulation of the expression above as a sum over permutations, one has the implementation in permutation_montecarlo_shapley() with the option to pass an early stopping strategy to reduce computation as done in Truncated MonteCarlo Shapley (TMCS).</p> <p>Also see</p> <p>It is also possible to use group_testing_shapley() to reduce the number of evaluations of the utility. The method is however typically outperformed by others in this module.</p> <p>Also see</p> <p>Additionally, you can consider grouping your data points using GroupedDataset and computing the values of the groups instead. This is not to be confused with \"group testing\" as implemented in group_testing_shapley(): any of the algorithms mentioned above, including Group Testing, can work to valuate groups of samples as units.</p>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley","title":"permutation_montecarlo_shapley","text":"<pre><code>permutation_montecarlo_shapley(\n    u: Utility,\n    done: StoppingCriterion,\n    *,\n    truncation: TruncationPolicy = NoTruncation(),\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes an approximate Shapley value by sampling independent permutations of the index set, approximating the sum:</p> \\[ v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} \\tilde{w}( | \\sigma_{:i} | )[u(\\sigma_{:i} \\cup \\{i\\}) \u2212 u(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears (see [[data-valuation]] for details).</p> <p>This implements the method described in (Ghorbani and Zou, 2019)<sup>1</sup> with a double stopping criterion.</p> <p>Todo</p> <p>Think of how to add Robin-Gelman or some other more principled stopping criterion.</p> <p>Instead of naively implementing the expectation, we sequentially add points to coalitions from a permutation and incrementally compute marginal utilities. We stop computing marginals for a given permutation based on a TruncationPolicy. (Ghorbani and Zou, 2019)<sup>1</sup> mention two policies: one that stops after a certain fraction of marginals are computed, implemented in FixedTruncation, and one that stops if the last computed utility (\"score\") is close to the total utility using the standard deviation of the utility as a measure of proximity, implemented in BootstrapTruncation.</p> <p>We keep sampling permutations and updating all shapley values until the StoppingCriterion returns <code>True</code>.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>function checking whether computation must stop.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>truncation</code> <p>An optional callable which decides whether to interrupt processing a permutation and set all subsequent marginals to zero. Typically used to stop computation when the marginal is small.</p> <p> TYPE: <code>TruncationPolicy</code> DEFAULT: <code>NoTruncation()</code> </p> <code>n_jobs</code> <p>number of jobs across which to distribute the computation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/shapley/montecarlo.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef permutation_montecarlo_shapley(\n    u: Utility,\n    done: StoppingCriterion,\n    *,\n    truncation: TruncationPolicy = NoTruncation(),\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    r\"\"\"Computes an approximate Shapley value by sampling independent\n    permutations of the index set, approximating the sum:\n\n    $$\n    v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)}\n    \\tilde{w}( | \\sigma_{:i} | )[u(\\sigma_{:i} \\cup \\{i\\}) \u2212 u(\\sigma_{:i})],\n    $$\n\n    where $\\sigma_{:i}$ denotes the set of indices in permutation sigma before\n    the position where $i$ appears (see [[data-valuation]] for details).\n\n    This implements the method described in (Ghorbani and Zou, 2019)&lt;sup&gt;&lt;a\n    href=\"#ghorbani_data_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt; with a double stopping criterion.\n\n    !!! Todo\n        Think of how to add Robin-Gelman or some other more principled stopping\n        criterion.\n\n    Instead of naively implementing the expectation, we sequentially add points\n    to coalitions from a permutation and incrementally compute marginal utilities.\n    We stop computing marginals for a given permutation based on a\n    [TruncationPolicy][pydvl.value.shapley.truncated.TruncationPolicy].\n    (Ghorbani and Zou, 2019)&lt;sup&gt;&lt;a href=\"#ghorbani_data_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n    mention two policies: one that stops after a certain\n    fraction of marginals are computed, implemented in\n    [FixedTruncation][pydvl.value.shapley.truncated.FixedTruncation],\n    and one that stops if the last computed utility (\"score\") is close to the\n    total utility using the standard deviation of the utility as a measure of\n    proximity, implemented in\n    [BootstrapTruncation][pydvl.value.shapley.truncated.BootstrapTruncation].\n\n    We keep sampling permutations and updating all shapley values\n    until the [StoppingCriterion][pydvl.value.stopping.StoppingCriterion] returns\n    `True`.\n\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: function checking whether computation must stop.\n        truncation: An optional callable which decides whether to interrupt\n            processing a permutation and set all subsequent marginals to\n            zero. Typically used to stop computation when the marginal is small.\n        n_jobs: number of jobs across which to distribute the computation.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    algorithm = \"permutation_montecarlo_shapley\"\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n    u = parallel_backend.put(u)\n    max_workers = parallel_backend.effective_n_jobs(n_jobs)\n    n_submitted_jobs = 2 * max_workers  # number of jobs in the executor's queue\n\n    seed_sequence = ensure_seed_sequence(seed)\n    result = ValuationResult.zeros(\n        algorithm=algorithm, indices=u.data.indices, data_names=u.data.data_names\n    )\n\n    pbar = tqdm(disable=not progress, total=100, unit=\"%\")\n\n    with parallel_backend.executor(\n        max_workers=max_workers, cancel_futures=CancellationPolicy.ALL\n    ) as executor:\n        pending: set[Future] = set()\n        while True:\n            pbar.n = 100 * done.completion()\n            pbar.refresh()\n\n            completed, pending = wait(pending, timeout=1.0, return_when=FIRST_COMPLETED)\n            for future in completed:\n                result += future.result()\n                # we could check outside the loop, but that means more\n                # submissions if the stopping criterion is unstable\n                if done(result):\n                    return result\n\n            # Ensure that we always have n_submitted_jobs in the queue or running\n            n_remaining_slots = n_submitted_jobs - len(pending)\n            seeds = seed_sequence.spawn(n_remaining_slots)\n            for i in range(n_remaining_slots):\n                future = executor.submit(\n                    _permutation_montecarlo_one_step,\n                    u,\n                    truncation,\n                    algorithm,\n                    seed=seeds[i],\n                )\n                pending.add(future)\n</code></pre>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley","title":"combinatorial_montecarlo_shapley","text":"<pre><code>combinatorial_montecarlo_shapley(\n    u: Utility,\n    done: StoppingCriterion,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Computes an approximate Shapley value using the combinatorial definition:</p> \\[v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)]\\] <p>This consists of randomly sampling subsets of the power set of the training indices in u.data, and computing their marginal utilities. See Data valuation for details.</p> <p>Note that because sampling is done with replacement, the approximation is poor even for \\(2^{m}\\) subsets with \\(m&gt;n\\), even though there are \\(2^{n-1}\\) subsets for each \\(i\\). Prefer permutation_montecarlo_shapley().</p> <p>Parallelization is done by splitting the set of indices across processes and computing the sum over subsets \\(S \\subseteq N \\setminus \\{i\\}\\) separately.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion for the computation.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>n_jobs</code> <p>number of parallel jobs across which to distribute the computation. Each worker receives a chunk of indices</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/shapley/montecarlo.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef combinatorial_montecarlo_shapley(\n    u: Utility,\n    done: StoppingCriterion,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n    r\"\"\"Computes an approximate Shapley value using the combinatorial\n    definition:\n\n    $$v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}}\n    \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)]$$\n\n    This consists of randomly sampling subsets of the power set of the training\n    indices in [u.data][pydvl.utils.utility.Utility], and computing their\n    marginal utilities. See [Data valuation][data-valuation] for details.\n\n    Note that because sampling is done with replacement, the approximation is\n    poor even for $2^{m}$ subsets with $m&gt;n$, even though there are $2^{n-1}$\n    subsets for each $i$. Prefer\n    [permutation_montecarlo_shapley()][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].\n\n    Parallelization is done by splitting the set of indices across processes and\n    computing the sum over subsets $S \\subseteq N \\setminus \\{i\\}$ separately.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        done: Stopping criterion for the computation.\n        n_jobs: number of parallel jobs across which to distribute the\n            computation. Each worker receives a chunk of\n            [indices][pydvl.utils.dataset.Dataset.indices]\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    map_reduce_job: MapReduceJob[NDArray, ValuationResult] = MapReduceJob(\n        u.data.indices,\n        map_func=_combinatorial_montecarlo_shapley,\n        reduce_func=lambda results: reduce(operator.add, results),\n        map_kwargs=dict(u=u, done=done, progress=progress),\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n    )\n    return map_reduce_job(seed=seed)\n</code></pre>"},{"location":"api/pydvl/value/shapley/naive/","title":"Naive","text":""},{"location":"api/pydvl/value/shapley/naive/#pydvl.value.shapley.naive","title":"pydvl.value.shapley.naive","text":"<p>This module implements exact Shapley values using either the combinatorial or permutation definition.</p> <p>The exact computation of \\(n\\) values takes \\(\\mathcal{O}(2^n)\\) evaluations of the utility and is therefore only possible for small datasets. For larger datasets, consider using any of the approximations, such as Monte Carlo, or proxy models like kNN.</p> <p>See Data valuation for details.</p>"},{"location":"api/pydvl/value/shapley/naive/#pydvl.value.shapley.naive.permutation_exact_shapley","title":"permutation_exact_shapley","text":"<pre><code>permutation_exact_shapley(\n    u: Utility, *, progress: bool = True\n) -&gt; ValuationResult\n</code></pre> <p>Computes the exact Shapley value using the formulation with permutations:</p> \\[v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{i-1} \\cup {i}) \u2212 u(\\sigma_{i})].\\] <p>See Data valuation for details.</p> <p>When the length of the training set is &gt; 10 this prints a warning since the computation becomes too expensive. Used mostly for internal testing and simple use cases. Please refer to the Monte Carlo approximations for practical applications.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/naive.py</code> <pre><code>def permutation_exact_shapley(u: Utility, *, progress: bool = True) -&gt; ValuationResult:\n    r\"\"\"Computes the exact Shapley value using the formulation with permutations:\n\n    $$v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{i-1}\n    \\cup {i}) \u2212 u(\\sigma_{i})].$$\n\n    See [Data valuation][data-valuation] for details.\n\n    When the length of the training set is &gt; 10 this prints a warning since the\n    computation becomes too expensive. Used mostly for internal testing and\n    simple use cases. Please refer to the [Monte Carlo\n    approximations][pydvl.value.shapley.montecarlo] for practical applications.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        progress: Whether to display progress bars for each job.\n\n    Returns:\n        Object with the data values.\n    \"\"\"\n\n    n = len(u.data)\n    # Note that the cache in utility saves most of the refitting because we\n    # use frozenset for the input.\n    if n &gt; 10:\n        warnings.warn(\n            f\"Large dataset! Computation requires {n}! calls to utility()\",\n            RuntimeWarning,\n        )\n\n    values = np.zeros(n)\n    for p in tqdm(\n        permutations(u.data.indices),\n        disable=not progress,\n        desc=\"Permutation\",\n        total=math.factorial(n),\n    ):\n        for i, idx in enumerate(p):\n            values[idx] += u(p[: i + 1]) - u(p[:i])\n    values /= math.factorial(n)\n\n    return ValuationResult(\n        algorithm=\"permutation_exact_shapley\",\n        status=Status.Converged,\n        values=values,\n        data_names=u.data.data_names,\n    )\n</code></pre>"},{"location":"api/pydvl/value/shapley/naive/#pydvl.value.shapley.naive.combinatorial_exact_shapley","title":"combinatorial_exact_shapley","text":"<pre><code>combinatorial_exact_shapley(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False\n) -&gt; ValuationResult\n</code></pre> <p>Computes the exact Shapley value using the combinatorial definition.</p> \\[v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)].\\] <p>See Data valuation for details.</p> <p>Note</p> <p>If the length of the training set is &gt; n_jobs*20 this prints a warning because the computation is very expensive. Used mostly for internal testing and simple use cases. Please refer to the Monte Carlo approximations for practical applications.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/shapley/naive.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef combinatorial_exact_shapley(\n    u: Utility,\n    *,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n) -&gt; ValuationResult:\n    r\"\"\"Computes the exact Shapley value using the combinatorial definition.\n\n    $$v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}}\n    \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)].$$\n\n    See [Data valuation][data-valuation] for details.\n\n    !!! Note\n        If the length of the training set is &gt; n_jobs*20 this prints a warning\n        because the computation is very expensive. Used mostly for internal\n        testing and simple use cases. Please refer to the\n        [Monte Carlo][pydvl.value.shapley.montecarlo] approximations for\n        practical applications.\n\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_jobs: Number of parallel jobs to use\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n    \"\"\"\n    # Arbitrary choice, will depend on time required, caching, etc.\n    if len(u.data) // n_jobs &gt; 20:\n        warnings.warn(\n            f\"Large dataset! Computation requires 2^{len(u.data)} calls to model.fit()\"\n        )\n\n    def reduce_fun(results: List[NDArray]) -&gt; NDArray:\n        return np.array(results).sum(axis=0)  # type: ignore\n\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    map_reduce_job: MapReduceJob[NDArray, NDArray] = MapReduceJob(\n        u.data.indices,\n        map_func=_combinatorial_exact_shapley,\n        map_kwargs=dict(u=u, progress=progress),\n        reduce_func=reduce_fun,\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n    )\n    values = map_reduce_job()\n    return ValuationResult(\n        algorithm=\"combinatorial_exact_shapley\",\n        status=Status.Converged,\n        values=values,\n        data_names=u.data.data_names,\n    )\n</code></pre>"},{"location":"api/pydvl/value/shapley/owen/","title":"Owen","text":""},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen","title":"pydvl.value.shapley.owen","text":""},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen--references","title":"References","text":"<ol> <li> <p>Okhrati, R., Lipani, A., 2021. A Multilinear Sampling Algorithm to Estimate Shapley Values. In: 2020 25th International Conference on Pattern Recognition (ICPR), pp. 7992\u20137999. IEEE.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen.OwenAlgorithm","title":"OwenAlgorithm","text":"<p>             Bases: <code>Enum</code></p> <p>Choices for the Owen sampling method.</p> ATTRIBUTE DESCRIPTION <code>Standard</code> <p>Use q \u2208 [0, 1]</p> <p> </p> <code>Antithetic</code> <p>Use q \u2208 [0, 0.5] and correlated samples</p> <p> </p>"},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen.owen_sampling_shapley","title":"owen_sampling_shapley","text":"<pre><code>owen_sampling_shapley(\n    u: Utility,\n    n_samples: int,\n    max_q: int,\n    *,\n    method: OwenAlgorithm = OwenAlgorithm.Standard,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult\n</code></pre> <p>Owen sampling of Shapley values as described in (Okhrati and Lipani, 2021)<sup>1</sup>.</p> <p>This function computes a Monte Carlo approximation to</p> \\[v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{\\backslash \\{i\\}})} [u(S \\cup \\{i\\}) - u(S)]\\] <p>using one of two methods. The first one, selected with the argument <code>mode = OwenAlgorithm.Standard</code>, approximates the integral with:</p> \\[\\hat{v}_u(i) = \\frac{1}{Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m \\cup \\{i\\}) - u(S^{(q_j)}_m)],\\] <p>where \\(q_j = \\frac{j}{Q} \\in [0,1]\\) and the sets \\(S^{(q_j)}\\) are such that a sample \\(x \\in S^{(q_j)}\\) if a draw from a \\(Ber(q_j)\\) distribution is 1.</p> <p>The second method, selected with the argument <code>mode = OwenAlgorithm.Antithetic</code>, uses correlated samples in the inner sum to reduce the variance:</p> \\[\\hat{v}_u(i) = \\frac{1}{2 Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m \\cup \\{i\\}) - u(S^{(q_j)}_m) + u((S^{(q_j)}_m)^c \\cup \\{i\\}) - u((S^{( q_j)}_m)^c)],\\] <p>where now \\(q_j = \\frac{j}{2Q} \\in [0,\\frac{1}{2}]\\), and \\(S^c\\) is the complement of \\(S\\).</p> <p>Note</p> <p>The outer integration could be done instead with a quadrature rule.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object holding data, model and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Numer of sets to sample for each value of q</p> <p> TYPE: <code>int</code> </p> <code>max_q</code> <p>Number of subdivisions for q \u2208 [0,1] (the element sampling probability) used to approximate the outer integral.</p> <p> TYPE: <code>int</code> </p> <code>method</code> <p>Selects the algorithm to use, see the description. Either OwenAlgorithm.Full for \\(q \\in [0,1]\\) or OwenAlgorithm.Halved for \\(q \\in [0,0.5]\\) and correlated samples</p> <p> TYPE: <code>OwenAlgorithm</code> DEFAULT: <code>Standard</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use. Each worker receives a chunk of the total of <code>max_q</code> values for q.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>parallel_backend</code> <p>Parallel backend instance to use for parallelizing computations. If <code>None</code>, use JoblibParallelBackend backend. See the Parallel Backends package for available options.</p> <p> TYPE: <code>Optional[ParallelBackend]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>Optional[ParallelConfig]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.3.0</p> <p>Changed in version 0.5.0</p> <p>Support for parallel computation and enable antithetic sampling.</p> <p>Changed in version 0.9.0</p> <p>Deprecated <code>config</code> argument and added a <code>parallel_backend</code> argument to allow users to pass the Parallel Backend instance directly.</p> Source code in <code>src/pydvl/value/shapley/owen.py</code> <pre><code>@deprecated(\n    target=True,\n    args_mapping={\"config\": \"config\"},\n    deprecated_in=\"0.9.0\",\n    remove_in=\"0.10.0\",\n)\ndef owen_sampling_shapley(\n    u: Utility,\n    n_samples: int,\n    max_q: int,\n    *,\n    method: OwenAlgorithm = OwenAlgorithm.Standard,\n    n_jobs: int = 1,\n    parallel_backend: Optional[ParallelBackend] = None,\n    config: Optional[ParallelConfig] = None,\n    progress: bool = False,\n    seed: Optional[Seed] = None\n) -&gt; ValuationResult:\n    r\"\"\"Owen sampling of Shapley values as described in\n    (Okhrati and Lipani, 2021)&lt;sup&gt;&lt;a href=\"#okhrati_multilinear_2021\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n\n    This function computes a Monte Carlo approximation to\n\n    $$v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{\\backslash \\{i\\}})}\n    [u(S \\cup \\{i\\}) - u(S)]$$\n\n    using one of two methods. The first one, selected with the argument ``mode =\n    OwenAlgorithm.Standard``, approximates the integral with:\n\n    $$\\hat{v}_u(i) = \\frac{1}{Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m\n    \\cup \\{i\\}) - u(S^{(q_j)}_m)],$$\n\n    where $q_j = \\frac{j}{Q} \\in [0,1]$ and the sets $S^{(q_j)}$ are such that a\n    sample $x \\in S^{(q_j)}$ if a draw from a $Ber(q_j)$ distribution is 1.\n\n    The second method, selected with the argument ``mode =\n    OwenAlgorithm.Antithetic``, uses correlated samples in the inner sum to\n    reduce the variance:\n\n    $$\\hat{v}_u(i) = \\frac{1}{2 Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m\n    \\cup \\{i\\}) - u(S^{(q_j)}_m) + u((S^{(q_j)}_m)^c \\cup \\{i\\}) - u((S^{(\n    q_j)}_m)^c)],$$\n\n    where now $q_j = \\frac{j}{2Q} \\in [0,\\frac{1}{2}]$, and $S^c$ is the\n    complement of $S$.\n\n    !!! Note\n        The outer integration could be done instead with a quadrature rule.\n\n    Args:\n        u: [Utility][pydvl.utils.utility.Utility] object holding data, model\n            and scoring function.\n        n_samples: Numer of sets to sample for each value of q\n        max_q: Number of subdivisions for q \u2208 [0,1] (the element sampling\n            probability) used to approximate the outer integral.\n        method: Selects the algorithm to use, see the description. Either\n            [OwenAlgorithm.Full][pydvl.value.shapley.owen.OwenAlgorithm] for\n            $q \\in [0,1]$ or\n            [OwenAlgorithm.Halved][pydvl.value.shapley.owen.OwenAlgorithm] for\n            $q \\in [0,0.5]$ and correlated samples\n        n_jobs: Number of parallel jobs to use. Each worker receives a chunk\n            of the total of `max_q` values for q.\n        parallel_backend: Parallel backend instance to use\n            for parallelizing computations. If `None`,\n            use [JoblibParallelBackend][pydvl.parallel.backends.JoblibParallelBackend] backend.\n            See the [Parallel Backends][pydvl.parallel.backends] package\n            for available options.\n        config: (**DEPRECATED**) Object configuring parallel computation,\n            with cluster address, number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n\n    Returns:\n        Object with the data values.\n\n    !!! tip \"New in version 0.3.0\"\n\n    !!! tip \"Changed in version 0.5.0\"\n        Support for parallel computation and enable antithetic sampling.\n\n    !!! tip \"Changed in version 0.9.0\"\n        Deprecated `config` argument and added a `parallel_backend`\n        argument to allow users to pass the Parallel Backend instance\n        directly.\n\n    \"\"\"\n    parallel_backend = _maybe_init_parallel_backend(parallel_backend, config)\n\n    map_reduce_job: MapReduceJob[NDArray, ValuationResult] = MapReduceJob(\n        u.data.indices,\n        map_func=_owen_sampling_shapley,\n        reduce_func=lambda results: reduce(operator.add, results),\n        map_kwargs=dict(\n            u=u,\n            method=OwenAlgorithm(method),\n            n_samples=n_samples,\n            max_q=max_q,\n            progress=progress,\n        ),\n        n_jobs=n_jobs,\n        parallel_backend=parallel_backend,\n    )\n\n    return map_reduce_job(seed=seed)\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/","title":"Truncated","text":""},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated","title":"pydvl.value.shapley.truncated","text":""},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy","title":"TruncationPolicy","text":"<pre><code>TruncationPolicy()\n</code></pre> <p>             Bases: <code>ABC</code></p> <p>A policy for deciding whether to stop computing marginals in a permutation.</p> <p>Statistics are kept on the number of calls and truncations as <code>n_calls</code> and <code>n_truncations</code> respectively.</p> ATTRIBUTE DESCRIPTION <code>n_calls</code> <p>Number of calls to the policy.</p> <p> TYPE: <code>int</code> </p> <code>n_truncations</code> <p>Number of truncations made by the policy.</p> <p> TYPE: <code>int</code> </p> <p>Todo</p> <p>Because the policy objects are copied to the workers, the statistics are not accessible from the coordinating process. We need to add methods for this.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self) -&gt; None:\n    self.n_calls: int = 0\n    self.n_truncations: int = 0\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy.reset","title":"reset  <code>abstractmethod</code>","text":"<pre><code>reset(u: Optional[Utility] = None)\n</code></pre> <p>Reset the policy to a state ready for a new permutation.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>@abc.abstractmethod\ndef reset(self, u: Optional[Utility] = None):\n    \"\"\"Reset the policy to a state ready for a new permutation.\"\"\"\n    ...\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy.__call__","title":"__call__","text":"<pre><code>__call__(idx: int, score: float) -&gt; bool\n</code></pre> <p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n    \"\"\"Check whether the computation should be interrupted.\n\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\n    ret = self._check(idx, score)\n    self.n_calls += 1\n    self.n_truncations += 1 if ret else 0\n    return ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.NoTruncation","title":"NoTruncation","text":"<pre><code>NoTruncation()\n</code></pre> <p>             Bases: <code>TruncationPolicy</code></p> <p>A policy which never interrupts the computation.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self) -&gt; None:\n    self.n_calls: int = 0\n    self.n_truncations: int = 0\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.NoTruncation.__call__","title":"__call__","text":"<pre><code>__call__(idx: int, score: float) -&gt; bool\n</code></pre> <p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n    \"\"\"Check whether the computation should be interrupted.\n\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\n    ret = self._check(idx, score)\n    self.n_calls += 1\n    self.n_truncations += 1 if ret else 0\n    return ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.FixedTruncation","title":"FixedTruncation","text":"<pre><code>FixedTruncation(u: Utility, fraction: float)\n</code></pre> <p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation after computing a fixed number of marginals.</p> <p>The experiments in Appendix B of (Ghorbani and Zou, 2019)<sup>1</sup> show that when the training set size is large enough, one can simply truncate the iteration over permutations after a fixed number of steps. This happens because beyond a certain number of samples in a training set, the model becomes insensitive to new ones. Alas, this strongly depends on the data distribution and the model and there is no automatic way of estimating this number.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>fraction</code> <p>Fraction of marginals in a permutation to compute before stopping (e.g. 0.5 to compute half of the marginals).</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, fraction: float):\n    super().__init__()\n    if fraction &lt;= 0 or fraction &gt; 1:\n        raise ValueError(\"fraction must be in (0, 1]\")\n    self.max_marginals = len(u.data) * fraction\n    self.count = 0\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.FixedTruncation.__call__","title":"__call__","text":"<pre><code>__call__(idx: int, score: float) -&gt; bool\n</code></pre> <p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n    \"\"\"Check whether the computation should be interrupted.\n\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\n    ret = self._check(idx, score)\n    self.n_calls += 1\n    self.n_truncations += 1 if ret else 0\n    return ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.RelativeTruncation","title":"RelativeTruncation","text":"<pre><code>RelativeTruncation(u: Utility, rtol: float)\n</code></pre> <p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation if the marginal utility is too low.</p> <p>This is called \"performance tolerance\" in (Ghorbani and Zou, 2019)<sup>1</sup>.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>rtol</code> <p>Relative tolerance. The permutation is broken if the last computed utility is less than <code>total_utility * rtol</code>.</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, rtol: float):\n    super().__init__()\n    self.rtol = rtol\n    logger.info(\"Computing total utility for permutation truncation.\")\n    self.total_utility = self.reset(u)\n    self._u = u\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.RelativeTruncation.__call__","title":"__call__","text":"<pre><code>__call__(idx: int, score: float) -&gt; bool\n</code></pre> <p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n    \"\"\"Check whether the computation should be interrupted.\n\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\n    ret = self._check(idx, score)\n    self.n_calls += 1\n    self.n_truncations += 1 if ret else 0\n    return ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.BootstrapTruncation","title":"BootstrapTruncation","text":"<pre><code>BootstrapTruncation(u: Utility, n_samples: int, sigmas: float = 1)\n</code></pre> <p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation if the last computed utility is close to the total utility, measured as a multiple of the standard deviation of the utilities.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Number of bootstrap samples to use to compute the variance of the utilities.</p> <p> TYPE: <code>int</code> </p> <code>sigmas</code> <p>Number of standard deviations to use as a threshold.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, n_samples: int, sigmas: float = 1):\n    super().__init__()\n    self.n_samples = n_samples\n    logger.info(\"Computing total utility for permutation truncation.\")\n    self.total_utility = u(u.data.indices)\n    self.count: int = 0\n    self.variance: float = 0\n    self.mean: float = 0\n    self.sigmas: float = sigmas\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.BootstrapTruncation.__call__","title":"__call__","text":"<pre><code>__call__(idx: int, score: float) -&gt; bool\n</code></pre> <p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n    \"\"\"Check whether the computation should be interrupted.\n\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\n    ret = self._check(idx, score)\n    self.n_calls += 1\n    self.n_truncations += 1 if ret else 0\n    return ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/types/","title":"Types","text":""},{"location":"api/pydvl/value/shapley/types/#pydvl.value.shapley.types","title":"pydvl.value.shapley.types","text":""},{"location":"api/pydvl/value/shapley/types/#pydvl.value.shapley.types.ShapleyMode","title":"ShapleyMode","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Supported algorithms for the computation of Shapley values.</p> <p>Todo</p> <p>Make algorithms register themselves here.</p>"},{"location":"examples/","title":"Examples","text":""},{"location":"examples/#data-valuation","title":"Data valuation","text":"<ul> <li> <p>Shapley values</p> <p>An introduction using the spotify dataset, showcasing grouped datasets and applied to improving model performance and identifying bogus data.</p> <p></p> </li> <li> <p>KNN Shapley</p> <p>A showcase of a fast model-specific valuation method using the iris dataset. </p> <p></p> </li> <li> <p>Data utility learning</p> <p>Learning a utility function from a few evaluations and using it to estimate the value of the remaining data.</p> <p></p> </li> <li> <p>Least Core</p> <p>An alternative solution concept from game theory, illustrated on a classification problem.</p> <p></p> </li> <li> <p>Data OOB</p> <p>A different and fast strategy for data valuation, using the out-of-bag error of a bagging model.</p> <p></p> </li> <li> <p>Faster Banzhaf values</p> <p>Using Banzhaf values to estimate the value of data points in MNIST, and evaluating convergence speed of MSR. </p> <p></p> </li> </ul>"},{"location":"examples/#influence-functions","title":"Influence functions","text":"<ul> <li> <p>For CNNs</p> <p>Detecting corrupted labels with influence functions on the ImageNet dataset.</p> <p></p> </li> <li> <p>For language models</p> <p>Using the IMDB dataset for sentiment analysis and a fine-tuned BERT model.</p> <p></p> </li> <li> <p>For mislabeled data</p> <p>Detecting corrupted labels using a synthetic dataset.</p> <p></p> </li> <li> <p>For outlier detection</p> <p>Using the wine dataset</p> <p></p> </li> </ul>"},{"location":"examples/data_oob/","title":"Data OOB","text":"<p>     This notebook introduces the Data-           OOB          method, an implementation based on a publication from Kwon and Zou \"           Data-             OOB            : Out-of-bag Estimate as a Simple and Efficient Data Value          \" ICML 2023 , using pyDVL.    </p> <p>     The objective of this paper is mainly to overcome the computational bottleneck of shapley-based data valuation methods that require to fit a significant number of models to accurately estimate marginal contributions. The algorithms compute data values from out of bag estimates using a bagging model.    </p> <p>     The value can be interpreted as a partition of the           OOB          estimate, which is originally introduced to estimate the prediction error. This           OOB          estimate is given as:    </p>      \\[ \\sum_{i=1}^n\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i, \\hat{f}_b(x_i))}{\\sum_{b=1}^{B} \\mathbb{1} (w_{bi}=0)} \\]     <pre><code>%autoreload\nfrom pydvl.utils import Dataset, Scorer, Seed, Utility, ensure_seed_sequence\nfrom pydvl.value import ValuationResult, compute_data_oob\n</code></pre> <p>     We will work with the           adult classification dataset          from the UCI repository. The objective is to predict whether a person earns more than 50k a year based on a set of features such as age, education, occupation, etc.    </p> <p>     With a helper function we download the data and obtain the following pandas dataframe, where the categorical features have been removed:    </p> <pre>\n<code>Found cached file: adult_data.pkl.\n</code>\n</pre> <pre><code>data_adult.head()\n</code></pre>            age                      fnlwgt                      education-num                      capital-gain                      capital-loss                      hours-per-week                      income                      0                      39                      77516                      13                      2174                      0                      40                      &lt;=50K                      1                      50                      83311                      13                      0                      0                      13                      &lt;=50K                      2                      38                      215646                      9                      0                      0                      40                      &lt;=50K                      3                      53                      234721                      7                      0                      0                      40                      &lt;=50K                      4                      28                      338409                      13                      0                      0                      40                      &lt;=50K           <pre><code>data = Dataset.from_arrays(\n    X=data_adult.drop(columns=[\"income\"]).values,\n    y=data_adult.loc[:, \"income\"].cat.codes.values,\n    random_state=random_state,\n)\n\nmodel = KNeighborsClassifier(n_neighbors=5)\n\nutility = Utility(model, data, Scorer(\"accuracy\", default=0.0))\n</code></pre> <pre><code>n_estimators = [100, 500]\noob_values = [\n    compute_data_oob(utility, n_est=n_est, max_samples=0.95, seed=random_state)\n    for n_est in n_estimators\n]\n</code></pre> <p>     The two results are stored in an array of           ValuationResult          objects. Here's their distribution. The left-hand side depicts value as it increases with rank and a 99% t-confidence interval. The right-hand side shows the histogram of values.    </p> <p>     Observe how adding estimators reduces the variance of the values, but doesn't change their distribution much.    </p>"},{"location":"examples/data_oob/#bagging-for-data-valuation","title":"Bagging for data valuation","text":""},{"location":"examples/data_oob/#setup","title":"Setup","text":"<p>     We begin by importing the main libraries and setting some defaults.    </p>      If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/data_oob/#computing-the-oob-values","title":"Computing the           OOB          values","text":"<p>     The main idea of Data-           OOB          is to take an existing classifier or regression model and compute a per-sample out-of-bag performance estimate via bagging.    </p> <p>     For this example, we use a simple KNN classifier with           \\(k=5\\)          neighbours on the data and compute the data-oob values with two choices for the number of estimators in the bagging. For that we construct a           Utility          object using the           Scorer          class to specify the metric to use for the evaluation. Note how we pass a random seed to           Dataset.from_arrays          in order to ensure that we always get the same split when running this notebook multiple times. This will be particularly important when running the standard point removal experiments later.    </p> <p>     We then use the           compute_data_oob          function to compute the data-oob values.    </p>"},{"location":"examples/data_oob/#point-removal-experiments","title":"Point removal experiments","text":"<p>     The standard procedure for the evaluation of data valuation schemes is the point removal experiment. The objective is to measure the evolution of performance when the best/worst points are removed from the training set. This can be done with the function           compute_removal_score          , which takes precomputed values and computes the performance of the model as points are removed.    </p> <p>     In order to test the true performance of DataOOB, we repeat the whole task of computing the values and the point removal experiment multiple times, including the splitting of the dataset into training and valuation sets. It is important to remember to pass random state adequately for full reproducibility.    </p>"},{"location":"examples/influence_imagenet/","title":"For CNNs","text":"If you are reading this in the documentation, some boilerplate has been omitted for convenience.     <pre><code>from pydvl.influence.torch import CgInfluence\nfrom pydvl.reporting.plots import plot_influence_distribution_by_label\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score\n</code></pre> <pre><code>label_names = {90: \"tables\", 100: \"boats\"}\ntrain_ds, val_ds, test_ds = load_preprocess_imagenet(\n    train_size=0.8,\n    test_size=0.1,\n    keep_labels=label_names,\n    downsampling_ratio=1,\n)\n\nprint(\"Normalised image dtype:\", train_ds[\"normalized_images\"][0].dtype)\nprint(\"Label type:\", type(train_ds[\"labels\"][0]))\nprint(\"Image type:\", type(train_ds[\"images\"][0]))\ntrain_ds.info()\n</code></pre> <p>     Let's take a closer look at a few image samples    </p> <p>     Let's now further pre-process the data and prepare for model training. The helper function     <code>      process_io     </code>     converts the normalized images into tensors and the labels to the indices 0 and 1 to train the classifier.    </p> <pre><code>def process_io(df: pd.DataFrame, labels: dict) -&amp;gt; Tuple[torch.Tensor, torch.Tensor]:\n    x = df[\"normalized_images\"]\n    y = df[\"labels\"]\n    ds_label_to_model_label = {\n        ds_label: idx for idx, ds_label in enumerate(labels.values())\n    }\n    x_nn = torch.stack(x.tolist()).to(DEVICE)\n    y_nn = torch.tensor([ds_label_to_model_label[yi] for yi in y], device=DEVICE)\n    return x_nn, y_nn\n\n\ntrain_x, train_y = process_io(train_ds, label_names)\nval_x, val_y = process_io(val_ds, label_names)\ntest_x, test_y = process_io(test_ds, label_names)\n\nbatch_size = 768\ntrain_data = DataLoader(TensorDataset(train_x, train_y), batch_size=batch_size)\ntest_data = DataLoader(TensorDataset(test_x, test_y), batch_size=batch_size)\nval_data = DataLoader(TensorDataset(val_x, val_y), batch_size=batch_size)\n</code></pre> <pre><code>device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel_ft = new_resnet_model(output_size=len(label_names))\nmgr = TrainingManager(\n    \"model_ft\",\n    model_ft,\n    nn.CrossEntropyLoss(),\n    train_data,\n    val_data,\n    MODEL_PATH,\n    device=device,\n)\n# Set use_cache=False to retrain the model\ntrain_loss, val_loss = mgr.train(n_epochs=50, use_cache=True)\n</code></pre> <pre><code>plot_losses(Losses(train_loss, val_loss))\n</code></pre> <p>     The confusion matrix and           \\(F_1\\)          score look good, especially considering the low resolution of the images and their complexity (they contain different objects)    </p> <pre><code>pred_y_test = np.argmax(model_ft(test_x).cpu().detach(), axis=1).cpu()\nmodel_score = f1_score(test_y.cpu(), pred_y_test, average=\"weighted\")\n\ncm = confusion_matrix(test_y.cpu(), pred_y_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_names.values())\nprint(\"f1_score of model:\", model_score)\ndisp.plot();\n</code></pre> <pre>\n<code>f1_score of model: 0.9062805208898536\n</code>\n</pre> <pre><code>influence_model = CgInfluence(mgr.model, mgr.loss, hessian_reg, progress=True)\ninfluence_model = influence_model.fit(train_data)\n</code></pre> <p>     On the instantiated influence object, we can call the method           influences          , which takes some test data and some input dataset with labels (which typically is the training data, or a subset of it). The influence type will be     <code>      up     </code>     . The other option,     <code>      perturbation     </code>     , is beyond the scope of this notebook, but more info can be found in the notebook           using the Wine dataset          or in the documentation for pyDVL.    </p> <pre><code>influences = influence_model.influences(test_x, test_y, train_x, train_y, mode=\"up\")\n</code></pre> <p>     The output  is a matrix of size     <code>      test_set_length     </code>     x     <code>      training_set_length     </code>     . Each row represents a test data point, and each column a training data point, so that entry           \\((i,j)\\)          represents the influence of training point           \\(j\\)          on test point           \\(i\\)          .    </p> <p>     Now we plot the histogram of the influence that all training images have on the image selected above, separated by their label.    </p> <p>     Rather unsurprisingly, the training points with the highest influence have the same label. Now we can take the training images with the same label and show those with highest and lowest scores.    </p> <p>     Looking at the images, it is difficult to explain why those on the right are more influential than those on the left. At first sight, the choice seems to be random (or at the very least noisy). Let's dig in a bit more by looking at average influences:    </p> <pre><code>avg_influences = np.mean(influences.cpu().numpy(), axis=0)\n</code></pre> <p>     Once again, let's plot the histogram of influence values by label.    </p> <p>     Next, for each class (you can change value by changing label key) we can have a look at the top and bottom images by average influence, i.e. we can show the images that have the highest and lowest average influence over all test images.    </p> <p>     Once again, it is not easy to explain why the images on the left have a lower influence than the ones on the right.    </p> <pre><code>corrupted_model = new_resnet_model(output_size=len(label_names))\ncorrupted_dataset, corrupted_indices = corrupt_imagenet(\n    dataset=train_ds,\n    fraction_to_corrupt=0.1,\n    avg_influences=avg_influences,\n)\n\ncorrupted_train_x, corrupted_train_y = process_io(corrupted_dataset, label_names)\ncorrupted_data = DataLoader(\n    TensorDataset(corrupted_train_x, corrupted_train_y), batch_size=batch_size\n)\n\nmgr = TrainingManager(\n    \"corrupted_model\",\n    corrupted_model,\n    nn.CrossEntropyLoss(),\n    corrupted_data,\n    val_data,\n    MODEL_PATH,\n    device=device,\n)\ntraining_loss, validation_loss = mgr.train(n_epochs=50, use_cache=True)\n</code></pre> <pre><code>plot_losses(Losses(training_loss, validation_loss))\n</code></pre> <pre>\n<code>F1 score of model with corrupted data: 0.8541666666666666\n</code>\n</pre> <p>     Interestingly, despite being trained on a corrupted dataset, the model has a fairly high           \\(F_1\\)          score. Let's now calculate the influence of the corrupted training data points over the test data points.    </p> <pre><code>influence_model = CgInfluence(mgr.model, mgr.loss, hessian_reg, progress=True)\ninfluence_model = influence_model.fit(corrupted_data)\ninfluences = influence_model.influences(\n    test_x, test_y, corrupted_train_x, corrupted_train_y\n)\n</code></pre> <p>     As before, since we are interested in the average influence on the test dataset, we take the average of influences across rows, and then plot the highest and lowest influences for a chosen label    </p> <pre><code>avg_corrupted_influences = np.mean(influences.cpu().numpy(), axis=0)\n</code></pre> <p>     As expected, the samples with lowest (negative) influence for the label \"boats\" are those that have been corrupted: all the images on the left are tables! We can compare the average influence of corrupted data with non-corrupted ones    </p>            label                      avg_non_corrupted_infl                      avg_corrupted_infl                      score_diff                      0                      tables                      -0.405254                      -12.999691                      12.594438                      1                      boats                      -0.544211                      -13.080050                      12.535838           <p>     And indeed corrupted data have a more negative influence on average than clean ones!    </p> <p>     Despite this being a useful property, influence functions are known to be unreliable for tasks of data valuation, especially in deep learning where the fundamental assumption of the theory (convexity) is grossly violated. A lot of factors (e.g. the size of the network, the training process or the Hessian regularization term) can interfere with the computation, to the point that often the results that we obtain cannot be trusted. This has been extensively studied in the recent paper:    </p> <p>       Basu, S., P. Pope, and S. Feizi.             Influence Functions in Deep Learning Are Fragile.            International Conference on Learning Representations (ICLR). 2021          .    </p> <p>     Nevertheless, influence functions offer a relatively quick and mathematically rigorous way to evaluate (at first order) the importance of a training point for a model's prediction.    </p>"},{"location":"examples/influence_imagenet/#influence-functions-for-neural-networks","title":"Influence functions for neural networks","text":"<p>     This notebook explores the use of influence functions for convolutional neural networks. In           the first part          we will investigate the usefulness, or lack thereof, of influence functions for the interpretation of a classifier's outputs.    </p> <p>     For our study we choose a pre-trained ResNet18, fine-tuned on the           tiny-imagenet dataset          . This dataset was created for a Stanford course on           Deep Learning for Computer Vision          , and is a subset of the           famous ImageNet          with 200 classes instead of 1000, and images down-sampled to a lower resolution of 64x64 pixels.    </p> <p>     After tuning the last layers of the network, we will use           pyDVL          to find the most and the least influential training images for the test set. This can sometimes be used to explain inference errors, or to direct efforts during data collection, although we will face inconclusive results with our model and data. This illustrates well-known issues of influence functions for neural networks.    </p> <p>     However, in the           final part of the notebook          we will see that influence functions are an effective tool for finding anomalous or corrupted data points.    </p> <p>     We conclude with an           appendix          with some basic theoretical concepts used.    </p>"},{"location":"examples/influence_imagenet/#imports-and-setup","title":"Imports and setup","text":""},{"location":"examples/influence_imagenet/#loading-and-preprocessing-the-dataset","title":"Loading and preprocessing the dataset","text":"<p>     We pick two classes arbitrarily to work with: 90 and 100, corresponding respectively to dining tables, and boats in Venice (you can of course select any other two classes, or more of them, although that would imply longer training times and some modifications in the notebook below). The dataset is loaded with     <code>      load_preprocess_imagenet()     </code>     , which returns three pandas     <code>      DataFrames     </code>     with training, validation and test sets respectively. Each dataframe has three columns: normalized images, labels and the original images. Note that you can load a subset of the data decreasing downsampling_ratio.    </p>"},{"location":"examples/influence_imagenet/#model-definition-and-training","title":"Model definition and training","text":"<p>     We use a ResNet18 from     <code>      torchvision     </code>     with final layers modified for binary classification.    </p> <p>     For training, we use the convenience class     <code>      TrainingManager     </code>     which transparently handles persistence after training. It is not part of the main pyDVL package but just a way to reduce clutter in this notebook.    </p> <p>     We train the model for 50 epochs and save the results. Then we plot the train and validation loss curves.    </p>"},{"location":"examples/influence_imagenet/#influence-computation","title":"Influence computation","text":"<p>     Let's now calculate influences! The central interface for computing influences is           InfluenceFunctionModel          . Since Resnet18 is quite big, we pick the conjugate gradient implementation           CgInfluence          , which takes a trained           torch.nn.Module          , the training loss and the training data. Other important parameters are the Hessian regularization term, which should be chosen as small as possible for the computation to converge (further details on why this is important can be found in the           Appendix          ).    </p>"},{"location":"examples/influence_imagenet/#analysing-influences","title":"Analysing influences","text":"<p>     With the computed influences we can study single images or all of them together:    </p>"},{"location":"examples/influence_imagenet/#influence-on-a-single-test-image","title":"Influence on a single test image","text":"<p>     Let's take any image in the test set:    </p>"},{"location":"examples/influence_imagenet/#analysing-the-average-influence-on-test-samples","title":"Analysing the average influence on test samples","text":"<p>     By averaging across the rows of the influence matrix, we obtain the average influence of each training sample on the whole test set:    </p>"},{"location":"examples/influence_imagenet/#detecting-corrupted-data","title":"Detecting corrupted data","text":"<p>     After facing the shortcomings of influence functions for explaining decisions, we move to an application with clear-cut results. Influences can be successfully used to detect corrupted or mislabeled samples, making them an effective tool to \"debug\" training data.    </p> <p>     We begin by training a new model (with the same architecture as before) on a dataset with some corrupted labels. The method     <code>      get_corrupted_imagenet     </code>     will take the training dataset and corrupt a certain fraction of the labels by flipping them. We use the same number of epochs and optimizer as before.    </p>"},{"location":"examples/influence_imagenet/#theory-of-influence-functions-for-neural-networks","title":"Theory of influence functions for neural networks","text":"<p>     In this appendix we will briefly go through the basic ideas of influence functions adapted for neural networks as introduced in           Koh, Pang Wei, and Percy Liang.             \"Understanding Black-box Predictions via Influence Functions\"            International conference on machine learning. PMLR, 2017.      </p> <p>     Note however that this paper departs from the standard and established theory and notation for influence functions. For a rigorous introduction to the topic we recommend classical texts like           Hampel, Frank R., Elvezio M. Ronchetti, Peter J. Rousseeuw, and Werner A. Stahel. Robust Statistics: The Approach Based on Influence Functions. 1st edition. Wiley Series in Probability and Statistics. New York: Wiley-Interscience, 2005. https://doi.org/10.1002/9781118186435.      </p>"},{"location":"examples/influence_imagenet/#upweighting-points","title":"Upweighting points","text":"<p>     Let's start by considering some input space           \\(\\mathcal{X}\\)          to a model (e.g. images) and an output space           \\(\\mathcal{Y}\\)          (e.g. labels). Let's take           \\(z_i = (x_i, y_i)\\)          to be the           \\(i\\)          -th training point, and           \\(\\theta\\)          to be the (potentially highly) multi-dimensional parameters of the neural network (i.e.           \\(\\theta\\)          is a big array with very many parameters). We will indicate with           \\(L(z, \\theta)\\)          the loss of the model for point           \\(z\\)          and parameters           \\(\\theta\\)          . When training the model we minimize the loss over all points, i.e. the optimal parameters are calculated through gradient descent on the following formula:    </p>      \\[ \\hat{\\theta} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) \\]     <p>     where           \\(n\\)          is the total number of training data points.    </p> <p>     For notational  convenience, let's define    </p>      \\[ \\hat{\\theta}_{-z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{z_i \\ne z} L(z_i, \\theta) \\ , \\]     <p>     i.e.           \\(\\hat{\\theta}_{-z}\\)          are the model parameters that minimize the total loss when           \\(z\\)          is not in the training dataset.    </p> <p>     In order to check the impact of each training point on the model, we would need to calculate           \\(\\hat{\\theta}_{-z}\\)          for each           \\(z\\)          in the training dataset, thus re-training the model at least ~           \\(n\\)          times (more if model training is noisy). This is computationally very expensive, especially for big neural networks. To circumvent this problem, we can just calculate a first order approximation of           \\(\\hat{\\theta}\\)          . This can be done through single backpropagation and without re-training the full model.    </p> <p>     Let's define    </p>      \\[ \\hat{\\theta}_{\\epsilon, z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) + \\epsilon L(z, \\theta) \\ , \\]     <p>     which is the optimal           \\(\\hat{\\theta}\\)          if we were to up-weigh           \\(z\\)          by an amount           \\(\\epsilon\\)          .    </p> <p>     From a classical result (a simple derivation is available in Appendix A of Koh and Liang's paper), we know that:    </p>      \\[ \\frac{d \\ \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta L(z, \\hat{\\theta}) \\]     <p>     where           \\(H_{\\hat{\\theta}} = \\frac{1}{n} \\sum_{i=1}^n \\nabla_\\theta^2 L(z_i, \\hat{\\theta})\\)          is the Hessian of           \\(L\\)          . Importantly, notice that this expression is only valid when           \\(\\hat{\\theta}\\)          is a minimum of           \\(L\\)          , or otherwise           \\(H_{\\hat{\\theta}}\\)          cannot be inverted!    </p>"},{"location":"examples/influence_imagenet/#approximating-the-influence-of-a-point","title":"Approximating the influence of a point","text":"<p>     We will define the influence of training point           \\(z\\)          on test point           \\(z_{\\text{test}}\\)          as           \\(\\mathcal{I}(z, z_{\\text{test}}) =  L(z_{\\text{test}}, \\hat{\\theta}_{-z}) - L(z_{\\text{test}}, \\hat{\\theta})\\)          (notice that it is higher for points           \\(z\\)          which positively impact the model score, since if they are excluded, the loss is higher). In practice, however, we will always use the infinitesimal approximation           \\(\\mathcal{I}_{up}(z, z_{\\text{test}})\\)          , defined as    </p>      \\[  \\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\frac{d L(z_{\\text{test}}, \\hat{\\theta}_{\\epsilon, z})}{d \\epsilon} \\Big|_{\\epsilon=0} \\]     <p>     Using the chain rule and the results calculated above, we thus have:    </p>      \\[  \\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_\\theta L(z, \\hat{\\theta}) \\]     <p>     In order to calculate this expression we need the gradient and the Hessian of the loss wrt. the model parameters           \\(\\hat{\\theta}\\)          . This can be easily done through a single backpropagation pass.    </p>"},{"location":"examples/influence_imagenet/#regularizing-the-hessian","title":"Regularizing the Hessian","text":"<p>     One very important assumption that we make when approximating influence is that           \\(\\hat{\\theta}\\)          is at least a local minimum of the loss. However, we clearly cannot guarantee this except for convex models, and despite good apparent convergence,           \\(\\hat{\\theta}\\)          might be located in a region with flat curvature or close to a saddle point. In particular, the Hessian might have vanishing eigenvalues making its direct inversion impossible.    </p> <p>     To circumvent this problem, instead of inverting the true Hessian           \\(H_{\\hat{\\theta}}\\)          , one can invert a small perturbation thereof:           \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\)          , with           \\(\\mathbb{I}\\)          being the identity matrix. This standard trick ensures that the eigenvalues of           \\(H_{\\hat{\\theta}}\\)          are bounded away from zero and therefore the matrix is invertible. In order for this regularization not to corrupt the outcome too much, the parameter           \\(\\lambda\\)          should be as small as possible while still allowing a reliable inversion of           \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\)          .    </p>"},{"location":"examples/influence_sentiment_analysis/","title":"For language models","text":"<p>     This notebooks showcases the use of influence functions for large language models. In particular, it focuses on sentiment analysis using the           IMDB dataset          and a fine-tuned           BERT          model.    </p> <p>     Not all the methods for influence function calculation can scale to large models and datasets. In this notebook we will use the           Kronecker-Factored Approximate Curvature          method, which is the only one that can scale to current state-of-the-art language models.    </p> <p>     The notebook is structured as follows:    </p> <ul> <li>        Setup            imports the required libraries and downloads the dataset and the model.     </li> <li>        Sentiment analysis            loads the model and the dataset and goes through a few examples of sentiment analysis.     </li> <li>        Model and data preparation            prepares the model and the dataset for influence function calculation. In particular, it assigns all the linear layers to require gradients and wraps the model so that only logits are returned (and not the loss or attention masks).     </li> <li>        Influence function computation            : shows how to calculate the influence function for a few test and train examples.     </li> <li>        Analysis of influence values            : analyses the influence values, trying to extract general information about the model and how it is affected by corruption in the training data.     </li> <li>        Influence functions by layer            : since ekfac is based on a block diagonal approximation of the Fisher information matrix, we can compute the influence function separately for each layer of the neural network. This section shows how to do that and how to analyse the results.     </li> </ul> <p>     Finally, the           Appendix          shows how to select the Hessian regularization parameter to obtain the best influence function approximation.    </p>      If you are reading this in the documentation, some boilerplate has been omitted for convenience.     <p>     Let's start by importing the required libraries. If not already installed, you can install them with     <code>      pip install -r requirements-notebooks.txt     </code>     .    </p> <pre><code>import os\nfrom copy import deepcopy\nfrom typing import Sequence\n\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn.functional as F\nfrom datasets import load_dataset\nfrom IPython.display import HTML, display\nfrom sklearn.metrics import f1_score\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer\n\nfrom pydvl.influence.torch import EkfacInfluence\nfrom support.torch import ImdbDataset, ModelLogitsWrapper\n</code></pre> <p>     Sentiment analysis is the task of classifying a sentence as having a positive or negative sentiment. For example, the sentence \"I love this movie\" has a positive sentiment, while \"I hate this movie\" has a negative sentiment. In this notebook we will use the IMDB dataset, which contains 50,000 movie reviews with corresponding labels. The dataset is split into 25,000 reviews for training and 25,000 reviews for testing. The dataset is balanced, meaning that there are the same number of positive and negative reviews in the training and test set.    </p> <pre><code>imdb = load_dataset(\"imdb\")\n</code></pre> <p>     Let's print an example of review and its label    </p> <pre><code>sample_review = imdb[\"train\"].select([24])\n\nprint(f\"Here is a sample review with label {sample_review['label'][0]}: \\n\")\n\ndisplay(HTML(sample_review[\"text\"][0].split(\"&lt;br/&gt;\")[0]))\ndisplay(HTML(sample_review[\"text\"][0].split(\"&lt;br/&gt;\")[-1]))\n</code></pre> <pre>\n<code>Here is a sample review with label 0: \n\n</code>\n</pre>       Without wishing to be a killjoy, Brad Sykes is responsible for at least two of the most dull and clich\u00e9d films i've ever seen - this being one of them, and Camp Blood being another.            I bought this for \u00a31, but remember, you can't put a price on 71 minutes of your life. You'd do well to avoid this turkey, even at a bargain basement price.      <p>     The review is negative, and so label 0 is associated to negative sentiment.    </p> <p>     The model is a BERT model fine-tuned on the IMDB dataset. BERT is a large language model that has been pre-trained on a large corpus of text. The model was fine-tuned on the IMDB dataset by AssemblyAI and is available on the HuggingFace model hub. We also load its tokenizer, which is used to convert sentences into numeric tokens.    </p> <pre><code>tokenizer = AutoTokenizer.from_pretrained(\"assemblyai/distilbert-base-uncased-sst2\")\nmodel = AutoModelForSequenceClassification.from_pretrained(\n    \"assemblyai/distilbert-base-uncased-sst2\"\n)\n</code></pre> <p>     Even if the model is trained on movie reviews, it can be used to classify any sentence as positive or negative. Let's try it on a simple sentence created by us.    </p> <pre><code>example_phrase = (\n    \"Pydvl is the best data valuation library, and it is fully open-source!\"\n)\n\ntokenized_example = tokenizer(\n    [example_phrase],\n    return_tensors=\"pt\",\n    truncation=True,\n)\n\nmodel_output = model(\n    input_ids=tokenized_example.input_ids,\n)\n</code></pre> <p>     The model output is a     <code>      SequenceClassificationOutput     </code>     object, which contains the logits and other information.    </p> <pre>\n<code>Model Output:\n SequenceClassifierOutput(loss=None, logits=tensor([[-2.6237,  2.8350]], grad_fn=&lt;AddmmBackward0&gt;), hidden_states=None, attentions=None)\n</code>\n</pre> <p>     For calculating probabilities and for the influence functions we only need the logits. Then the softmax function converts the logits into probabilities.    </p> <pre><code>model_predictions = F.softmax(model_output.logits, dim=1)\n</code></pre> <p>     The model is quite confident that the sentence has a positive sentiment, which is correct.    </p> <pre>\n<code>Positive probability: 99.6%\nNegative probability: 0.4%\n</code>\n</pre> <p>     Let's examine the model's f1 score on a small subset of the test set.    </p> <pre><code>sample_test_set = imdb[\"test\"].shuffle(seed=seed).select(range(50 if not is_CI else 5))\nsample_test_set = sample_test_set.map(\n    lambda example: tokenizer(example[\"text\"], truncation=True, padding=\"max_length\"),\n    batched=True,\n)\nsample_test_set.set_format(\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])\nmodel.eval()\nwith torch.no_grad():\n    logits = model(\n        input_ids=sample_test_set[\"input_ids\"],\n        attention_mask=sample_test_set[\"attention_mask\"],\n    ).logits\n    predictions = torch.argmax(logits, dim=1)\n</code></pre> <pre><code>f1_score_value = f1_score(sample_test_set[\"label\"], predictions)\nprint(f\"F1 Score: {round(f1_score_value, 3)}\")\n</code></pre> <pre>\n<code>F1 Score: 0.955\n</code>\n</pre> <p>     In this section we will define two helper function and classes that will be used in the rest of the notebook.    </p> <pre><code>def print_sentiment_preds(\n    model: ModelLogitsWrapper, model_input: torch.Tensor, true_label: int\n):\n    \"\"\"\n    Prints the sentiment predictions in a human-readable format given a model and an\n    input. It also prints the true label.\n    \"\"\"\n    model_predictions = F.softmax(model(model_input.unsqueeze(0)), dim=1)\n    print(\n        \"Positive probability: \"\n        + str(round(model_predictions[0][1].item(), 3) * 100)\n        + \"%\"\n    )\n    print(\n        \"Negative probability: \"\n        + str(round(model_predictions[0][0].item(), 3) * 100)\n        + \"%\"\n    )\n\n    true_label = \"Positive\" if true_label == 1 else \"Negative\"\n    print(f\"True label: {true_label} \\n\")\n\n\ndef strip_layer_names(param_names: Sequence[str]):\n    \"\"\"\n    Helper function that strips the parameter names of the model and the transformer,\n    so that they can be printed and compared more easily.\n    \"\"\"\n    stripped_param_names = []\n    for name in param_names:\n        name = name.replace(\"model.\", \"\")\n        if name.startswith(\"distilbert.transformer.\"):\n            name = name.replace(\"distilbert.transformer.\", \"\")\n        stripped_param_names.append(name)\n    return stripped_param_names\n</code></pre> <p>     Importantly, we will need to assign all the linear layers to require gradients, so that we can compute the influence function with respect to them. Keep in mind that the current implementation of Ekfac only supports linear layers, so if any other type of layer in the model requires gradients the initialisation of the influence function class will fail.    </p> <pre><code>for param in model.named_parameters():\n    param[1].requires_grad = False\n\nfor m_name, module in model.named_modules():\n    if len(list(module.children())) == 0 and len(list(module.parameters())) &amp;gt; 0:\n        if isinstance(module, torch.nn.Linear):\n            for p_name, param in module.named_parameters():\n                if (\n                    (\"ffn\" in m_name and not is_CI)\n                    or \"pre_classifier\" in m_name\n                    or \"classifier\" in m_name\n                ):\n                    param.requires_grad = True\n</code></pre> <p>     Albeit restrictive, linear layers constitute a large fraction of the parameters of most large language models, and so our analysis still holds a lot of information about the full neural network.    </p> <pre>\n<code>Total parameters: 66.96 millions\nParameters requiring gradients: 28.93 millions\nRatio of Linear over other layer types: 43.20%\n</code>\n</pre> <p>     We are now ready to compute the influence function for a few testing and training examples. Let's start by selecting a subset of the full training and testing dataset and wrapping them in a     <code>      DataLoader     </code>     object, so that we can easily do batching.    </p> <pre><code>NUM_TRAIN_EXAMPLES = 100 if not is_CI else 7\nNUM_TEST_EXAMPLES = 100 if not is_CI else 5\n\nsmall_train_dataset = (\n    imdb[\"train\"]\n    .shuffle(seed=seed)\n    .select([i for i in list(range(NUM_TRAIN_EXAMPLES))])\n)\nsmall_test_dataset = (\n    imdb[\"test\"].shuffle(seed=seed).select([i for i in list(range(NUM_TEST_EXAMPLES))])\n)\n\ntrain_dataset = ImdbDataset(small_train_dataset, tokenizer=tokenizer)\ntest_dataset = ImdbDataset(small_test_dataset, tokenizer=tokenizer)\n\ntrain_dataloader = torch.utils.data.DataLoader(\n    train_dataset, batch_size=7, shuffle=True\n)\ntest_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=5, shuffle=True)\n</code></pre> <p>     For influence computation we need to take the model in evaluation mode, so that no dropout or batch normalization is applied. Then, we can fit the Ekfac representation.    </p> <pre><code>wrapped_model = ModelLogitsWrapper(model)\nwrapped_model.eval()\n\nekfac_influence_model = EkfacInfluence(\n    wrapped_model,\n    progress=True,\n)\nekfac_influence_model = ekfac_influence_model.fit(train_dataloader)\n</code></pre> <pre>\n<code>K-FAC blocks - batch progress:   0%|          | 0/15 [00:00&lt;?, ?it/s]</code>\n</pre> <p>     And the approximate Hessian is thus obtained. Considering that the model has almost 30 million parameters requiring gradients, this was very fast! Of course, this Hessian is computed using only a very small fraction (~0.4%) of the training data, and for a better approximation we should use a larger subset.    </p> <p>     Before continuing, we need to set the Hessian regularization parameter to an appropriate value. A way to decide which is better can be found in the           Appendix          . Here, we will just set it to 1e-5.    </p> <pre><code>ekfac_influence_model.hessian_regularization = 1e-5\n</code></pre> <p>     We calculate the influence of the first batch of training data over the first batch of test data. This is because influence functions are very expensive to compute, and so to keep the runtime of this notebook within a few minutes we need to restrict ourselves to a small number of examples.    </p> <pre><code>test_input, test_labels, test_text = next(iter(test_dataloader))\ntrain_input, train_labels, train_text = next(iter(train_dataloader))\n</code></pre> <p>     And let's finally compute the influence function values    </p> <pre><code>ekfac_train_influences = ekfac_influence_model.influences(\n    test_input,\n    test_labels,\n    train_input,\n    train_labels,\n)\n</code></pre> <pre>\n<code>/home/jakob/Documents/pyDVL/venv/lib/python3.10/site-packages/transformers/models/distilbert/modeling_distilbert.py:222: UserWarning: There is a performance drop because we have not yet implemented the batching rule for aten::masked_fill.Tensor. Please file us an issue on GitHub so that we can prioritize its implementation. (Triggered internally at ../aten/src/ATen/functorch/BatchedFallback.cpp:82.)\n  scores = scores.masked_fill(\n</code>\n</pre> <p>     Now that we have calculated the influences for a few examples, let's analyse some of the extreme values.    </p> <p>     Let's plot the influence values as a heatmap for easily spotting patterns.    </p> <p>     Most of the test and training examples have similar influence, close to zero. However, there is one test and one training samples that stand out. In particular, their cross influence is very large and negative. Let's examine them more closely.    </p> <pre>\n<code>Training example with idx 3: \n\nPositive probability: 18.099999999999998%\nNegative probability: 81.89999999999999%\nTrue label: Positive \n\nSentence:\n</code>\n</pre>       In the process of trying to establish the audiences' empathy with Jake Roedel (Tobey Maguire) the filmmakers slander the North and the Jayhawkers. Missouri never withdrew from the Union and the Union Army was not an invading force. The Southerners fought for State's Rights: the right to own slaves, elect crooked legislatures and judges, and employ a political spoils system. There's nothing noble in that. The Missourians could have easily traveled east and joined the Confederate Army.             It seems to me that the story has nothing to do with ambiguity. When Jake leaves the Bushwhackers, it's not because he saw error in his way, he certainly doesn't give himself over to the virtue of the cause of abolition.      <p>     We can see that, despite being positive, this review is quite hard to classify. Its language is overall negative, mostly associated to the facts narrated rather than the movie itself. Notice how several terms are related to war and invasion.    </p> <pre>\n<code>Test example with idx 4: \n\nPositive probability: 39.6%\nNegative probability: 60.4%\nTrue label: Negative \n\nSentence:\n</code>\n</pre>       \"An astronaut (Michael Emmet) dies while returning from a mission and his body is recovered by the military. The base where the dead astronaut is taken to becomes the scene of a bizarre invasion plan from outer space. Alien embryos inside the dead astronaut resurrect the corpse and begin a terrifying assault on the military staff in the hopes of conquering the world,\" according to the DVD sleeve's synopsis.             A Roger Corman \"American International\" production. The man who fell to Earth impregnated, Mr. Emmet (as John Corcoran), does all right. Angela Greene is his pretty conflicted fianc\u00e9e. And, Ed Nelson (as Dave Randall) is featured as prominently. With a bigger budget, better opening, and a re-write for crisper characterizations, this could have been something approaching classic 1950s science fiction.             *** Night of the Blood Beast (1958) Bernard L. Kowalski, Roger Corman ~ Michael Emmet, Angela Greene, Ed Nelson      <p>     This review is also quite hard to classify. This time it has a negative sentiment towards the movie, but it also contains several words with positive connotation. The parallel with the previous review is quite interesting since both talk about an invasion.    </p> <p>     As it is often the case when analysing influence functions, it is hard to understand why these examples have such a large influence. We have seen some interesting patterns, mostly related to similarities in the language and words used, but it is hard to say with certainty if these are the reasons for such a large influence.    </p> <p>     A           recent paper          has explored this topic in high detail, even for much larger language models than BERT (up to ~50 billion parameters!). Among the most interesting findings is that smaller models tend to rely a lot on word-to-word correspondencies, while larger models are more capable of extracting higher level concepts, drawing connections between words across multiple phrases.    </p> <p>     For more info, you can visit our           blog on influence functions for large language models      </p> <p>     In this sections we want to get an idea of how influence functions change when training examples are corrupted. In the next cell we will flip the label of all the training examples and compute the influences on the same test batch as before.    </p> <pre><code>modified_train_labels = deepcopy(train_labels)\nmodified_train_labels = 1 - train_labels\n\ncorrupted_ekfac_train_influences = ekfac_influence_model.influences(\n    test_input,\n    test_labels,\n    train_input,\n    modified_train_labels,\n)\n</code></pre> <p>     Overall, when corrupted the influences tend to become negative, as expected. Nevertheless, there are cases where values go from slightly negative to positive, mostly isolated to the second and last test samples. Single values can be quite noisy, so it is difficult to generalise this result, but it would be interesting to see how common these cases are in the full test dataset.    </p> <p>     Since ekfac is based on a block diagonal approximation of the Fisher information matrix, we can compute the influence functions separately for each layer of the neural network. In this section we show how to do that and we briefly analyse the results.    </p> <pre><code>influences_by_layer = ekfac_influence_model.influences_by_layer(\n    test_input,\n    test_labels,\n    train_input,\n    train_labels,\n)\n</code></pre> <p>     The method     <code>      influences_by_layer     </code>     returns a dictionary containing the influence function values for each layer of the neural network as a tensor. To recover the full influence values as returned by the     <code>      influences     </code>     (as done in the previous section), we need to sum each layer's values.    </p> <pre><code>influences = torch.zeros_like(ekfac_train_influences)\nfor layer_id, value in influences_by_layer.items():\n    influences += value.detach()\n</code></pre> <p>     And if we plot the result as a heatmap we can see that the results are the same as in           Negative influence training examples      </p> <p>     Let's analyse how the influence values change across different layers for given test and train examples.    </p> <p>     The plot above shows the influences for test idx 0 and all train idx apart idx=3 (excluded for clarity since it has a very large absolute value). We can see that the scores tend to keep their sign across layers, but in almost all cases tend to decrease when approaching the output layer. This is not always the case, and in fact other test examples show different patterns. Understanding why this happens is an interesting research direction.    </p> <p>     Ekfac is a powerful approximate method for computing the influence function of models that use a cross-entropy loss. In this notebook we applied it to sentiment analysis with BERT on the IMDB dataset. However, this method can be applied to much larger models and problems, e.g. to analyse the influence of entire sentences generated by GPT, Llama or Claude. For more info, you can visit our           paper pill on influence functions for large language models      </p> <p>     The Hessian regularization value impacts a lot the quality of the influence function approximation. In general, the value should be chosen as small as possible so that the results are finite. In practice, even when finite the influence values can be too large and lead to numerical instabilities. In this section we show how to efficiently analyse the impact of the Hessian regularization value with the ekfac method.    </p> <p>     Let's start with a few additional imports.    </p> <pre><code>import pandas as pd\nfrom scipy.stats import pearsonr, spearmanr\n</code></pre> <p>     The method     <code>      explore_hessian_regularization     </code>     will calculate the influence values of the training examples with each other for a range of Hessian regularization values. The method optimises gradient calculation and Hessian inversion to minimise the computation time.    </p> <pre><code>influences_by_reg_value = ekfac_influence_model.explore_hessian_regularization(\n    train_input,\n    train_labels,\n    regularization_values=[1e-15, 1e-9, 1e-5, 1],\n)\n</code></pre> <pre>\n<code>/home/jakob/Documents/pyDVL/venv/lib/python3.10/site-packages/transformers/models/distilbert/modeling_distilbert.py:222: UserWarning: There is a performance drop because we have not yet implemented the batching rule for aten::masked_fill.Tensor. Please file us an issue on GitHub so that we can prioritize its implementation. (Triggered internally at ../aten/src/ATen/functorch/BatchedFallback.cpp:82.)\n  scores = scores.masked_fill(\n</code>\n</pre> <p>     The resulting object,     <code>      influences_by_reg_value     </code>     is a dictionary that associates to each regularization value the influences for each layer of the neural network. This is a lot of data, so we will first organise it in a pandas dataframe and take the average across training examples.    </p> <pre><code>cols = [\"reg_value\", \"layer_id\", \"mean_infl\"]\ninfl_df = pd.DataFrame(influences_by_reg_value, columns=cols)\nfor reg_value in influences_by_reg_value:\n    for layer_id, layer_influences in influences_by_reg_value[reg_value].items():\n        mean_infl = torch.mean(layer_influences, dim=0).detach().numpy()\n        infl_df = pd.concat(\n            [infl_df, pd.DataFrame([[reg_value, layer_id, mean_infl]], columns=cols)]\n        )\n</code></pre> <pre>\n<code>/tmp/ipykernel_8503/1081261490.py:6: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n  infl_df = pd.concat(\n</code>\n</pre> <p>     With this dataframe, we can take contiguous values of regularization and, for each layer, calculate the Pearson and Spearman correlation coefficients. This will give us an idea of how the influence values change with the regularization value.    </p> <pre><code>result_corr = {}\nfor layer_id, group_df in infl_df.groupby(\"layer_id\"):\n    result_corr[layer_id + \"_pearson\"] = {}\n    result_corr[layer_id + \"_spearman\"] = {}\n    for idx, mean_infl in enumerate(group_df[\"mean_infl\"]):\n        if idx == 0:\n            continue\n        reg_value_diff = f\"Reg: {group_df['reg_value'].iloc[idx-1]} -&amp;gt; {group_df['reg_value'].iloc[idx]}\"\n        pearson = pearsonr(mean_infl, group_df[\"mean_infl\"].iloc[idx - 1]).statistic\n        spearman = spearmanr(mean_infl, group_df[\"mean_infl\"].iloc[idx - 1]).statistic\n        result_corr[layer_id + \"_pearson\"].update({f\"{reg_value_diff}\": pearson})\n        result_corr[layer_id + \"_spearman\"].update({f\"{reg_value_diff}\": spearman})\nresult_df = pd.DataFrame(result_corr).T\n</code></pre> <p>     Let's plot the correlations heatmap. The y-axis reports Spearman and Pearson correlations for each layer, while the x-axis reports pairs of regularization values. High correlations mean that influences are stable across regularization values.    </p> <p>     In our case, we can see that for regularization = 1 the spearman correlation becomes very bad. However, for a large range of regularization values smaller than 1 the sample rankings are stable. This is a good indicator that the model is not too sensitive to the regularization value. We therefore chose the value 1e-5 for our analysis.    </p>"},{"location":"examples/influence_sentiment_analysis/#influence-functions-for-large-language-models","title":"Influence functions for Large Language Models","text":""},{"location":"examples/influence_sentiment_analysis/#setup","title":"Setup","text":""},{"location":"examples/influence_sentiment_analysis/#sentiment-analysis","title":"Sentiment Analysis","text":""},{"location":"examples/influence_sentiment_analysis/#model-and-data-preparation","title":"Model and Data Preparation","text":""},{"location":"examples/influence_sentiment_analysis/#influence-function-computation","title":"Influence function computation","text":""},{"location":"examples/influence_sentiment_analysis/#analysis-of-influence-values","title":"Analysis of influence values","text":""},{"location":"examples/influence_sentiment_analysis/#negative-influence-training-examples","title":"Negative influence training examples","text":""},{"location":"examples/influence_sentiment_analysis/#influence-of-corrupted-training-examples","title":"Influence of corrupted training examples","text":""},{"location":"examples/influence_sentiment_analysis/#influence-functions-by-layer","title":"Influence functions by layer","text":""},{"location":"examples/influence_sentiment_analysis/#conclusion","title":"Conclusion","text":""},{"location":"examples/influence_sentiment_analysis/#appendix-choosing-the-hessian-regularization-value","title":"Appendix: Choosing the Hessian regularization value","text":""},{"location":"examples/influence_synthetic/","title":"For mislabeled data","text":"If you are reading this in the documentation, some boilerplate has been omitted for convenience.     <pre><code>%autoreload\n%matplotlib inline\n\nimport os\nimport random\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nfrom pydvl.influence.torch import DirectInfluence, CgInfluence\nfrom support.shapley import (\n    synthetic_classification_dataset,\n    decision_boundary_fixed_variance_2d,\n)\nfrom support.common import (\n    plot_gaussian_blobs,\n    plot_losses,\n    plot_influences,\n)\nfrom support.torch import (\n    fit_torch_model,\n    TorchLogisticRegression,\n)\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\nfrom torch.optim import AdamW, lr_scheduler\nfrom torch.utils.data import DataLoader, TensorDataset\n</code></pre> <p>     The following code snippet generates the aforementioned dataset.    </p> <pre><code>train_data, val_data, test_data = synthetic_classification_dataset(\n    means, sigma, num_samples, train_size=0.7, test_size=0.2\n)\n</code></pre> <p>     Given the simplicity of the dataset, we can calculate exactly the optimal decision boundary(that which maximizes our accuracy). The following code maps a continuous line of z values to a 2-dimensional vector in feature space (More details are in the appendix to this notebook.)    </p> <pre><code>decision_boundary_fn = decision_boundary_fixed_variance_2d(means[0], means[1])\ndecision_boundary = decision_boundary_fn(np.linspace(-1.5, 1.5, 100))\n</code></pre> <pre><code>plot_gaussian_blobs(\n    train_data,\n    test_data,\n    xlabel=\"$x_0$\",\n    ylabel=\"$x_1$\",\n    legend_title=\"$y - labels$\",\n    line=decision_boundary,\n    s=10,\n    suptitle=\"Plot of train-test data\",\n)\n</code></pre> <p>     Note that there are samples which go across the optimal decision boundary and will be wrongly labelled. The optimal decision boundary can not discriminate these as the mislabelling is a consequence of the presence of random noise.    </p> <pre><code>model = TorchLogisticRegression(num_features)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\nnum_epochs = 50\nlr = 0.05\nweight_decay = 0.05\nbatch_size = 256\n\ntrain_data_loader = DataLoader(\n    TensorDataset(\n        torch.as_tensor(train_data[0]),\n        torch.as_tensor(train_data[1], dtype=torch.float64).unsqueeze(-1),\n    ),\n    batch_size=batch_size,\n    shuffle=True,\n)\n\nval_data_loader = DataLoader(\n    TensorDataset(\n        torch.as_tensor(val_data[0]),\n        torch.as_tensor(val_data[1], dtype=torch.float64).unsqueeze(-1),\n    ),\n    batch_size=batch_size,\n    shuffle=True,\n)\n\noptimizer = AdamW(params=model.parameters(), lr=lr, weight_decay=weight_decay)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\nlosses = fit_torch_model(\n    model=model,\n    training_data=train_data_loader,\n    val_data=val_data_loader,\n    loss=F.binary_cross_entropy,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    num_epochs=num_epochs,\n    device=device,\n)\n</code></pre> <p>     And let's check that the model is not overfitting    </p> <pre><code>plot_losses(losses)\n</code></pre> <p>     A look at the confusion matrix also shows good results    </p> <p>     It is important that the model converges to a point near the optimum, since the influence values assume that we are at a minimum (or close) in the loss landscape. The function    </p>      \\[I(x_1, y_1, x_2, y_2) \\colon \\mathbb{R}^d \\times \\mathbb{R}^d \\to \\mathbb{R}\\]     <p>     measures the influence of the data point           \\(x_1\\)          onto           \\(x_2\\)          conditioned on the training targets           \\(y_1\\)          and           \\(y_2\\)          trough some model parameters           \\(\\theta\\)          . If the loss function L is differentiable, we can take           \\(I\\)          to be    </p> <p>     $$ I(x_1, x_2) = \\nabla_\\theta\\; L(x_1, y_1) ^\\mathsf{T} \\; H_\\theta^{-1} \\; \\nabla_\\theta \\; L(x_2, y_2) $$ See           \"Understanding Black-box Predictions via Influence Functions\"          for a detailed derivation of this formula    </p> <p>     Let's take a subset of the training data points, which we will calculate the influence values of.    </p> <pre><code>x = train_data[0][:100]\ny = train_data[1][:100]\n</code></pre> <p>     In pyDVL, the influence of the training points on the test points can be calculated with the following    </p> <pre><code>train_x = torch.as_tensor(x)\ntrain_y = torch.as_tensor(y, dtype=torch.float64).unsqueeze(-1)\ntest_x = torch.as_tensor(test_data[0])\ntest_y = torch.as_tensor(test_data[1], dtype=torch.float64).unsqueeze(-1)\n\ntrain_data_loader = DataLoader(\n    TensorDataset(train_x, train_y),\n    batch_size=batch_size,\n)\n\ninfluence_model = DirectInfluence(\n    model,\n    F.binary_cross_entropy,\n    hessian_regularization=0.0,\n)\ninfluence_model = influence_model.fit(train_data_loader)\n\ninfluence_values = influence_model.influences(\n    test_x, test_y, train_x, train_y, mode=\"up\"\n)\n</code></pre> <p>     The above explicitly constructs the Hessian. This can often be computationally expensive and conjugate gradient approximate calculation should be used for bigger models.    </p> <p>     With the influence type 'up', training influences have shape [NxM] where N is the number of test samples and M is the number of training samples. They therefore associate to each training sample its influence on each test sample.  Influence type 'perturbation', instead, return an array of shape  [NxMxF], where F is the number of features in input, ie. the length of x.    </p> <p>     In our case, in order to have a value of the total average influence of a point we can just average across training samples.    </p> <pre><code>mean_train_influences = np.mean(influence_values.cpu().numpy(), axis=0)\n</code></pre> <p>     Let's plot the results (adjust colorbar_limits for better color gradient)    </p> <pre><code>plot_influences(\n    x,\n    mean_train_influences,\n    line=decision_boundary,\n    xlabel=\"$x_0$\",\n    ylabel=\"$x_1$\",\n    suptitle=\"Influences of input points\",\n    legend_title=\"influence values\",\n    # colorbar_limits=(-0.3,),\n);\n</code></pre> <p>     We can see that, as we approach the separation line, the influences tend to move away from zero, i.e. the points become more decisive for model training, some in a positive way, some negative.    </p> <p>     As a further test, let's introduce some labelling errors into           \\(y\\)          and see how the distribution of the influences changes. Let's flip the first 10 labels and calculate influences    </p> <pre><code>y_corrupted = np.copy(y)\ny_corrupted[:10] = [1 - yi for yi in y[:10]]\ntrain_y_corrupted = torch.as_tensor(y_corrupted, dtype=torch.float64).unsqueeze(-1)\ntrain_corrupted_data_loader = DataLoader(\n    TensorDataset(\n        train_x,\n        train_y_corrupted,\n    ),\n    batch_size=batch_size,\n)\n\ninfluence_model = DirectInfluence(\n    model,\n    F.binary_cross_entropy,\n    hessian_regularization=0.0,\n)\ninfluence_model = influence_model.fit(train_corrupted_data_loader)\ninfluence_values = influence_model.influences(\n    test_x, test_y, train_x, train_y_corrupted, mode=\"up\"\n)\n\nmean_train_influences = np.mean(influence_values.cpu().numpy(), axis=0)\n</code></pre> <pre>\n<code>Average mislabelled data influence: -0.8450009471117079\nAverage correct data influence: 0.010396852920315886\n</code>\n</pre> <p>     Red circles indicate the points which have been corrupted. We can see that the mislabelled data have a more negative average influence on the model, especially those that are farther away from the decision boundary.    </p> <p>     The \"direct\" method that we have used above involves the inversion of the Hessian matrix of the model. If a model has           \\(n\\)          training points and           \\(\\theta  \\in \\mathbb{R}^p\\)          parameters, this requires           \\(O(n \\ p^2 + p^3)\\)          operations, which for larger models, like neural networks, becomes quickly unfeasible. Conjugate gradient avoids the explicit computation of the Hessian via a technique called implicit Hessian-vector products (HVPs), which typically takes           \\(O(n \\ p)\\)          operations.    </p> <p>     In the next cell we will use conjugate gradient to compute the influence factors. Since logistic regression is a very simple model, \"cg\" actually slows computation with respect to the direct method, which in this case is a much better choice. Nevertheless, we are able to verify that the influences calculated with \"cg\" are the same (to a minor error) as those calculated directly.    </p> <pre><code>influence_model = CgInfluence(\n    model,\n    F.binary_cross_entropy,\n    hessian_regularization=0.0,\n)\ninfluence_model = influence_model.fit(train_corrupted_data_loader)\ninfluence_values = influence_model.influences(\n    test_x, test_y, train_x, train_y_corrupted\n)\nmean_train_influences = np.mean(influence_values.cpu().numpy(), axis=0)\n\nprint(\"Average mislabelled data influence:\", np.mean(mean_train_influences[:10]))\nprint(\"Average correct data influence:\", np.mean(mean_train_influences[10:]))\n</code></pre> <pre>\n<code>Average mislabelled data influence: -0.8448414156979295\nAverage correct data influence: 0.010395021813591145\n</code>\n</pre> <p>     Averages are very similar to the ones calculated through direct method. Same is true for the plot    </p>"},{"location":"examples/influence_synthetic/#influence-functions-for-data-mislabeling","title":"Influence functions for data mislabeling","text":"<p>     In this notebook, we will take a closer look at the theory of influence functions with the help of a synthetic dataset. Data mislabeling occurs whenever some examples from a usually big dataset are wrongly-labeled. In real-life this happens fairly often, e.g. as a consequence of human error, or noise in the data.    </p> <p>     Let's consider a classification problem with the following notation:    </p>      \\[ \\begin{align*} x_i &amp;\\in \\mathbb{R}^d \\\\ y_i &amp;\\in \\{0, 1\\} \\\\ \\forall i &amp;\\in [ N ] \\end{align*} \\]     <p>     In other words, we have a dataset containing           \\(N\\)          samples, each with label 1 or 0. As typical example you can think of y indicating whether a patient has a disease based on some feature representation           \\(x\\)          .    </p> <p>     Let's now introduce a toy model that will help us delve into the theory and practical utility of influence functions. We will assume that           \\(y\\)          is a Bernoulli binary random variable while the input           \\(x\\)          is d-dimensional Gaussian distribution which depends on the label           \\(y\\)          . More precisely:    </p>      \\[ y_i \\sim \\text{Ber}\\left (0.5 \\right) \\\\ x_i \\sim \\mathcal{N}\\left ((1 - y_i) \\mu_1 + y_i \\mu_2, \\sigma^2 I \\right), \\]     <p>     with fixed means and diagonal covariance. Implementing the sampling scheme in python is straightforward and can be achieved by first sampling           \\(y\\)          and afterward           \\(x\\)          .    </p>"},{"location":"examples/influence_synthetic/#imports","title":"Imports","text":""},{"location":"examples/influence_synthetic/#dataset","title":"Dataset","text":""},{"location":"examples/influence_synthetic/#plotting-the-dataset","title":"Plotting the dataset","text":"<p>     Let's plot the dataset is plotted with their respective labels and the optimal decision line    </p>"},{"location":"examples/influence_synthetic/#training-the-model","title":"Training the model","text":"<p>     We will now train a logistic regression model on the training data. This can be done with the following    </p>"},{"location":"examples/influence_synthetic/#calculating-influences","title":"Calculating influences","text":""},{"location":"examples/influence_synthetic/#inversion-through-conjugate-gradient","title":"Inversion through conjugate gradient","text":""},{"location":"examples/influence_synthetic/#appendix-calculating-the-decision-boundary","title":"Appendix: Calculating the decision boundary","text":"<p>     For obtaining the optimal discriminator one has to solve the equation    </p>      \\[p(x|y=0)=p(x|y=1)\\]     <p>     and determine the solution set           \\(X\\)          . Let's take the following probabilities    </p>      \\[ \\begin{align*} p(x|y=0)&amp;=\\mathcal{N}\\left (\\mu_1, \\sigma^2 I \\right) \\\\ p(x|y=1)&amp;=\\mathcal{N}\\left (\\mu_2, \\sigma^2 I \\right) \\end{align*} \\]     <p>     For a single fixed diagonal variance parameterized by           \\(\\sigma\\)          , the optimal discriminator lays at points which are equidistant from the means of the two distributions, i.e.    </p>      \\[ \\begin{align*} \\| x - \\mu_1 \\|^2 &amp;= \\| x - \\mu_2 \\|^2 \\\\ \\| \\mu_1 \\|^2 -2 x^\\mathsf{T} \\mu_1 &amp;= \\| \\mu_2 \\|^2 -2 x^\\mathsf{T} \\mu_2 \\\\ \\implies 0 &amp;= 2 (\\mu_2 - \\mu_1)^\\mathsf{T} x + \\| \\mu_1 \\|^2 - \\| \\mu_2 \\|^2 \\\\ 0 &amp;= \\mu_1^\\mathsf{T}x - \\mu_2^\\mathsf{T}x - \\frac{1}{2} \\mu_1^\\mathsf{T} \\mu_1 + \\frac{1}{2} \\mu_2^\\mathsf{T} \\mu_2 \\end{align*} \\]     <p>     This is just the implicit description of the line. Solving for the explicit form can be achieved by enforcing a functional form           \\(f(z) = x = a z + b\\)          with           \\(z \\in \\mathbb{R}\\)          onto           \\(x\\)          . After the term is inserted in the previous equation    </p>      \\[ 0 = (\\mu_2 - \\mu_1)^\\mathsf{T} (az + b) + \\frac{1}{2} \\| \\mu_1 \\|^2 - \\| \\mu_2 \\|^2 \\]     <p>     We can write           \\(a\\)          since, by symmetry, it is expected to be explicitly orthogonal to           \\(\\mu_2 - \\mu_1\\)          . Then, solving for           \\(b\\)          , the solution can be found to be    </p>      \\[ f(z) = \\underbrace{\\begin{bmatrix} 0 &amp; 1 \\\\ -1 &amp; 0 \\end{bmatrix} (\\mu_2 - \\mu_1)}_a z + \\underbrace{\\frac{\\mu_1 + \\mu_2}{2}}_b \\]"},{"location":"examples/influence_wine/","title":"For outlier detection","text":"If you are reading this in the documentation, some boilerplate has been omitted for convenience.     <p>     Let's start by loading the imports, the dataset and splitting it into train, validation and test sets. We will use a large test set to have a less noisy estimate of the average influence.    </p> <pre><code>%autoreload\n%matplotlib inline\n\nimport os\nimport random\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nfrom support.common import plot_losses\nfrom support.torch import TorchMLP, fit_torch_model\nfrom pydvl.influence.torch import (\n    DirectInfluence,\n    CgInfluence,\n    ArnoldiInfluence,\n    EkfacInfluence,\n    NystroemSketchInfluence,\n    LissaInfluence,\n)\nfrom support.shapley import load_wine_dataset\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score\nfrom torch.optim import Adam, lr_scheduler\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom scipy.stats import pearsonr, spearmanr\n</code></pre> <pre><code>training_data, val_data, test_data, feature_names = load_wine_dataset(\n    train_size=0.6, test_size=0.3\n)\n</code></pre> <p>     We will corrupt some of the training points by flipping their labels    </p> <pre><code>num_corrupted_idxs = 10\ntraining_data[1][:num_corrupted_idxs] = torch.tensor(\n    [(val + 1) % 3 for val in training_data[1][:num_corrupted_idxs]]\n)\n</code></pre> <p>     and let's wrap it in a pytorch data loader    </p> <pre><code>training_data_loader = DataLoader(\n    TensorDataset(*training_data), batch_size=32, shuffle=False\n)\nval_data_loader = DataLoader(TensorDataset(*val_data), batch_size=32, shuffle=False)\ntest_data_loader = DataLoader(TensorDataset(*test_data), batch_size=32, shuffle=False)\n</code></pre> <pre><code>feature_dimension = 13\nnum_classes = 3\nnetwork_size = [16, 16]\nlayers_size = [feature_dimension, *network_size, num_classes]\nnum_epochs = 300\nlr = 0.005\nweight_decay = 0.01\n\nnn_model = TorchMLP(layers_size)\nnn_model.to(device)\n\noptimizer = Adam(params=nn_model.parameters(), lr=lr, weight_decay=weight_decay)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\n\nlosses = fit_torch_model(\n    model=nn_model,\n    training_data=training_data_loader,\n    val_data=val_data_loader,\n    loss=F.cross_entropy,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    num_epochs=num_epochs,\n    device=device,\n)\n</code></pre> <p>     Let's check that the training has found a stable minimum by plotting the training and validation loss    </p> <pre><code>plot_losses(losses)\n</code></pre> <p>     Since it is a classification problem, let's also take a look at the confusion matrix on the test set    </p> <p>     And let's compute the f1 score of the model    </p> <pre><code>f1_score(test_data[1], pred_y_test, average=\"weighted\")\n</code></pre> <pre>\n<code>0.9633110554163186</code>\n</pre> <p>     Let's now move to calculating influences of each point on the total score.    </p> <pre><code>influence_model = DirectInfluence(\n    nn_model,\n    F.cross_entropy,\n    hessian_regularization=0.1,\n)\ninfluence_model = influence_model.fit(training_data_loader)\ntrain_influences = influence_model.influences(*test_data, *training_data, mode=\"up\")\n</code></pre> <p>     the returned matrix, train_influences, has a quantity of columns equal to the points in the training set, and a number of rows equal to the points in the test set. At each element           \\(a_{i,j}\\)          it stores the influence that training point           \\(j\\)          has on the classification of test point           \\(i\\)          .    </p> <p>     If we take the average across every column of the influences matrix, we obtain an estimate of the overall influence of a training point on the total accuracy of the network.    </p> <pre><code>mean_train_influences = np.mean(train_influences.cpu().numpy(), axis=0)\n</code></pre> <p>     The following histogram shows that there are big differences in score within the training set (notice the log-scale on the y axis).    </p> <p>     We can see that the corrupted points tend to have a negative effect on the model, as expected    </p> <pre>\n<code>Average influence of corrupted points:  -1.0667533\nAverage influence of other points:  0.10814369\n</code>\n</pre> <p>     We have seen how to calculate the influence of single training points on each test point using influence_type 'up'. Using influence_type 'perturbation' we can also calculate the influence of the input features of each point. In the next cell we will calculate the average influence of each feature on training and test points, and ultimately assess which are the most relevant to model performance.    </p> <pre><code>influence_model.hessian_regularization = 1.0\nfeature_influences = influence_model.influences(\n    *test_data, *training_data, mode=\"perturbation\"\n)\n</code></pre> <p>     The explicit calculation of the Hessian matrix is numerically challenging, and due to the high memory need infeasible for larger models.  PyDVL allows to use several approximation methods for the action of the inverse Hessian matrix to overcome this bottleneck:    </p> <ul> <li>      Iteration-based:      <ul> <li>        Conjugate Gradients (Cg)       </li> <li>        Linear time Stochastic Second-Order Approximation (                 LiSSA                )       </li> </ul> </li> <li>      Low-rank Approximations:      <ul> <li>        Arnoldi       </li> <li>        Nystr\u00f6m Sketch-and-Solve (Nystr\u00f6m)       </li> </ul> </li> <li>      Factorization-based:      <ul> <li>        Eigenvalue-corrected Kronecker Factorization (                 EKFAC                )       </li> </ul> </li> </ul> <p>     In the following, we show the usage of these approximation methods and investigate their performance.    </p> <p>     Since the Hessian is symmetric and positive definite (at least after applying a sufficient regularization), we can utilize the           Conjugate Gradients Algorithm          to approximately solve the equations    </p>      \\[ (H + \\lambda \\operatorname{I}) x = b\\]     <p>     Most importantly, the algorithm do not require the computation of the full Hessian matrix, but only requires the implementation of Hessian vector products. pyDVL implements a stable block variant of preconditioned conjugate gradients algorithm.    </p> <pre><code>from pydvl.influence.torch.pre_conditioner import NystroemPreConditioner\n\nnn_model.to(\"cpu\")\ncg_influence_model = CgInfluence(\n    nn_model,\n    F.cross_entropy,\n    hessian_regularization=0.1,\n    progress=True,\n    use_block_cg=True,\n    pre_conditioner=NystroemPreConditioner(rank=5),\n)\ncg_influence_model = cg_influence_model.fit(training_data_loader)\ncg_train_influences = cg_influence_model.influences(\n    *test_data, *training_data, mode=\"up\"\n)\nmean_cg_train_influences = np.mean(cg_train_influences.numpy(), axis=0)\n</code></pre> <p>     Let's compare the results obtained through conjugate gradient with those from the direct method    </p> <pre>\n<code>Percentage error of Cg over direct method:20.877432823181152 %\n</code>\n</pre> <pre>\n<code>Pearson Correlation Cg vs direct 0.9977952163687263\nSpearman Correlation Cg vs direct 0.9972793913897776\n</code>\n</pre> <p>     The           LiSSA          method is a stochastic approximation of the inverse Hessian vector product. Compared to conjugate gradient it is faster but less accurate and typically suffers from instability.    </p> <p>     In order to find the solution of the HVP,           LiSSA          iteratively approximates the inverse of the Hessian matrix with the following update:    </p>      \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\]     <p>     where           \\(d\\)          and           \\(s\\)          are a dampening and a scaling factor.    </p> <pre><code>lissa_influence_model = LissaInfluence(\n    nn_model,\n    F.cross_entropy,\n    hessian_regularization=0.1,\n    progress=True,\n)\nlissa_influence_model = lissa_influence_model.fit(training_data_loader)\nlissa_train_influences = lissa_influence_model.influences(\n    *test_data, *training_data, mode=\"up\"\n)\nmean_lissa_train_influences = np.mean(lissa_train_influences.numpy(), axis=0)\n</code></pre> <pre>\n<code>Percentage error of Lissa over direct method:9.416493028402328 %\n</code>\n</pre> <pre>\n<code>Pearson Correlation Lissa vs direct 0.999796846529674\nSpearman Correlation Lissa vs direct 0.9990729778068873\n</code>\n</pre> <p>     The Arnoldi method leverages a low rank approximation of the Hessian matrix to reduce the memory requirements. It is generally much faster than the conjugate gradient method and can achieve similar accuracy.    </p> <pre><code>arnoldi_influence_model = ArnoldiInfluence(\n    nn_model,\n    F.cross_entropy,\n    rank_estimate=30,\n    hessian_regularization=0.1,\n)\narnoldi_influence_model = arnoldi_influence_model.fit(training_data_loader)\narnoldi_train_influences = arnoldi_influence_model.influences(\n    *test_data, *training_data, mode=\"up\"\n)\nmean_arnoldi_train_influences = np.mean(arnoldi_train_influences.numpy(), axis=0)\n</code></pre> <pre>\n<code>Percentage error of Arnoldi over direct method:37.59587109088898 %\n</code>\n</pre> <pre>\n<code>Pearson Correlation Arnoldi vs direct 0.9873076667742712\nSpearman Correlation Arnoldi vs direct 0.9791621533113334\n</code>\n</pre> <p>     Similar to the Arnoldi method. the Nystr\u00f6m method uses a low-rank approximation, which is computed from random projections of the Hessian matrix. In general the approximation is expected to be worse then the Arnoldi approximation, but is cheaper to compute.    </p> <pre><code>nystroem_influence_model = NystroemSketchInfluence(\n    nn_model,\n    F.cross_entropy,\n    rank=30,\n    hessian_regularization=0.1,\n)\nnystroem_influence_model = nystroem_influence_model.fit(training_data_loader)\nnystroem_train_influences = nystroem_influence_model.influences(\n    *test_data, *training_data, mode=\"up\"\n)\nmean_nystroem_train_influences = np.mean(nystroem_train_influences.numpy(), axis=0)\n</code></pre> <pre>\n<code>Percentage error of Nystr\u00f6m over direct method:37.205782532691956 %\n</code>\n</pre> <pre>\n<code>Pearson Correlation Nystr\u00f6m vs direct 0.9946487475679316\nSpearman Correlation Nystr\u00f6m vs direct 0.985500163740333\n</code>\n</pre> <p>     The           EKFAC          method is a more recent technique that leverages the Kronecker product structure of the Hessian matrix to reduce the memory requirements. It is generally much faster than iterative methods like conjugate gradient and Arnoldi and it allows for an easier handling of memory. Therefore, it is the only technique that can scale to very large models (e.g. billions of parameters). Its accuracy is however much worse. Let's see how it performs on our example.    </p> <pre><code>ekfac_influence_model = EkfacInfluence(\n    nn_model,\n    update_diagonal=True,\n    hessian_regularization=0.1,\n)\nekfac_influence_model = ekfac_influence_model.fit(training_data_loader)\nekfac_train_influences = ekfac_influence_model.influences(\n    *test_data, *training_data, mode=\"up\"\n)\nmean_ekfac_train_influences = np.mean(ekfac_train_influences.numpy(), axis=0)\n</code></pre> <pre>\n<code>Percentage error of EK-FAC over direct method:969.1289901733398 %\n</code>\n</pre> <p>     The accuracy is not good, and it is not recommended to use this method for small models. Nevertheless, a look at the actual influence values reveals that the EK-FAC estimates are not completely off.    </p> <p>     The above plot shows a good correlation between the EK-FAC and the direct method. Corrupted points have been circled in red, and in both the direct and approximate case they are correcly identified as having negative influence on the model's accuracy. This is confirmed by explicit calculation of the Pearson and Spearman correlation coefficients.    </p> <pre>\n<code>Pearson Correlation EK-FAC vs direct 0.9602782923532728\nSpearman Correlation EK-FAC vs direct 0.8999118321283724\n</code>\n</pre> <p>     The correlation between the EK-FAC and the direct method is quite good, and it improves significantly if we just keep top-20 highest absolute influences.    </p> <pre>\n<code>Pearson Correlation EK-FAC vs direct - top-20 influences 0.9876922383437592\nSpearman Correlation EK-FAC vs direct - top-20 influences 0.9338345864661652\n</code>\n</pre> <p>     When we calculate influence scores, typically we are more interested in assessing which training points have the highest or lowest impact on the model rather than having a precise estimate of the influence value. EK-FAC then provides a fast and memory-efficient way to calculate a coarse influence ranking of the training points which scales very well even to the largest neural networks.    </p> <p>     This was a quick introduction to the pyDVL interface for influence functions. Despite their speed and simplicity, influence functions are known to be a very noisy estimator of data quality, as pointed out in the paper           \"Influence functions in deep learning are fragile\"          . The size of the network, the weight decay, the inversion method used for calculating influences, the size of the test set: they all add up to the total amount of noise. Experiments may therefore give quantitative and qualitatively different results if not averaged across several realisations. Shapley values, on the contrary, have shown to be a more robust, but this comes at the cost of high computational requirements. PyDVL employs several parallelization and caching techniques to optimize such calculations.    </p>"},{"location":"examples/influence_wine/#influence-functions-for-outlier-detection","title":"Influence functions for outlier detection","text":"<p>     This notebook shows how to calculate influences on a NN model using pyDVL for an arbitrary dataset, and how this can be used to find anomalous or corrupted data points.    </p> <p>     It uses the wine dataset from sklearn: given a set of 13 different input parameters regarding a particular bottle, each related to some physical property (e.g. concentration of magnesium, malic acidity, alcoholic percentage, etc.), the model will need to predict to which of 3 classes the wine belongs to. For more details, please refer to the           sklearn documentation          .    </p>"},{"location":"examples/influence_wine/#imports","title":"Imports","text":""},{"location":"examples/influence_wine/#dataset","title":"Dataset","text":""},{"location":"examples/influence_wine/#fit-a-neural-network-to-the-data","title":"Fit a neural network to the data","text":"<p>     We will train a 2-layer neural network. PyDVL has some convenience wrappers to initialize a pytorch NN. If you already have a model loaded and trained, you can skip this section.    </p>"},{"location":"examples/influence_wine/#calculating-influences-for-small-neural-networks","title":"Calculating influences for small neural networks","text":"<p>     The following cell calculates the influences of each training data point on the neural network.  Neural networks have typically a very bumpy parameter space, which, during training, is explored until the configuration that minimises the loss is found. There is an important assumption in influence functions that the model lays at a (at least local) minimum of such loss, and if this is not fulfilled many issues can arise. In order to avoid this scenario, a regularisation term should be used whenever dealing with big and noisy models.    </p>"},{"location":"examples/influence_wine/#influence-of-training-features","title":"Influence of training features","text":""},{"location":"examples/influence_wine/#speeding-up-influences-for-big-models","title":"Speeding up influences for big models","text":""},{"location":"examples/influence_wine/#cg","title":"Cg","text":""},{"location":"examples/influence_wine/#lissa","title":"Lissa","text":""},{"location":"examples/influence_wine/#arnoldi","title":"Arnoldi","text":""},{"location":"examples/influence_wine/#nystrom","title":"Nystr\u00f6m","text":""},{"location":"examples/influence_wine/#ekfac","title":"EKFAC","text":""},{"location":"examples/influence_wine/#conclusions","title":"Conclusions","text":""},{"location":"examples/least_core_basic/","title":"Least Core","text":"<p>     We will be using the following functions and classes from pyDVL.    </p> <pre><code>%autoreload\nfrom pydvl.utils import (\n    Dataset,\n    Utility,\n)\nfrom pydvl.value import compute_least_core_values, LeastCoreMode, ValuationResult\nfrom pydvl.reporting.plots import shaded_mean_std\nfrom pydvl.reporting.scores import compute_removal_score\n</code></pre> <pre><code>X, y = make_classification(\n    n_samples=dataset_size,\n    n_features=50,\n    n_informative=25,\n    n_classes=3,\n    random_state=random_state,\n)\n</code></pre> <pre><code>full_dataset = Dataset.from_arrays(\n    X, y, stratify_by_target=True, random_state=random_state\n)\nsmall_dataset = Dataset.from_arrays(\n    X,\n    y,\n    stratify_by_target=True,\n    train_size=train_size,\n    random_state=random_state,\n)\n</code></pre> <pre><code>model = LogisticRegression(max_iter=500, solver=\"liblinear\")\n</code></pre> <pre><code>model.fit(full_dataset.x_train, full_dataset.y_train)\nprint(\n    f\"Training accuracy: {100 * model.score(full_dataset.x_train, full_dataset.y_train):0.2f}%\"\n)\nprint(\n    f\"Testing accuracy: {100 * model.score(full_dataset.x_test, full_dataset.y_test):0.2f}%\"\n)\n</code></pre> <pre>\n<code>Training accuracy: 86.25%\nTesting accuracy: 70.00%\n</code>\n</pre> <pre><code>model.fit(small_dataset.x_train, small_dataset.y_train)\nprint(\n    f\"Training accuracy: {100 * model.score(small_dataset.x_train, small_dataset.y_train):0.2f}%\"\n)\nprint(\n    f\"Testing accuracy: {100 * model.score(small_dataset.x_test, small_dataset.y_test):0.2f}%\"\n)\n</code></pre> <pre>\n<code>Training accuracy: 100.00%\nTesting accuracy: 47.89%\n</code>\n</pre> <pre><code>utility = Utility(model=model, data=small_dataset)\n</code></pre> <pre><code>exact_values = compute_least_core_values(\n    u=utility,\n    mode=LeastCoreMode.Exact,\n    progress=True,\n)\n</code></pre> <pre><code>exact_values_df = exact_values.to_dataframe(column=\"exact_value\").T\nexact_values_df = exact_values_df[sorted(exact_values_df.columns)]\n</code></pre> <pre><code>budget_array = np.linspace(200, 2 ** len(small_dataset), num=10, dtype=int)\n\nall_estimated_values_df = []\nall_errors = {budget: [] for budget in budget_array}\n\nfor budget in tqdm(budget_array):\n    dfs = []\n    errors = []\n    column_name = f\"estimated_value_{budget}\"\n    for i in range(20):\n        values = compute_least_core_values(\n            u=utility,\n            mode=LeastCoreMode.MonteCarlo,\n            n_iterations=budget,\n            n_jobs=n_jobs,\n        )\n        df = (\n            values.to_dataframe(column=column_name)\n            .drop(columns=[f\"{column_name}_stderr\", f\"{column_name}_updates\"])\n            .T\n        )\n        df = df[sorted(df.columns)]\n        error = mean_squared_error(\n            exact_values_df.loc[\"exact_value\"].values, np.nan_to_num(df.values.ravel())\n        )\n        all_errors[budget].append(error)\n        df[\"budget\"] = budget\n        dfs.append(df)\n    estimated_values_df = pd.concat(dfs)\n    all_estimated_values_df.append(estimated_values_df)\n\nvalues_df = pd.concat(all_estimated_values_df)\nerrors_df = pd.DataFrame(all_errors)\n</code></pre> <p>     We can see that the approximation error decreases, on average, as the we increase the budget.    </p> <p>     Still, the decrease may not always necessarily happen when we increase the number of iterations because of the fact that we sample the subsets with replacement in the Monte Carlo method i.e there may be repeated subsets.    </p> <pre><code>utility = Utility(model=model, data=full_dataset)\n</code></pre> <pre><code>method_names = [\"Random\", \"Least Core\"]\nremoval_percentages = np.arange(0, 0.41, 0.05)\n</code></pre> <pre><code>all_scores = []\n\nfor i in trange(5):\n    for method_name in method_names:\n        if method_name == \"Random\":\n            values = ValuationResult.from_random(size=len(utility.data))\n        else:\n            values = compute_least_core_values(\n                u=utility,\n                mode=LeastCoreMode.MonteCarlo,\n                n_iterations=n_iterations,\n                n_jobs=n_jobs,\n            )\n        scores = compute_removal_score(\n            u=utility,\n            values=values,\n            percentages=removal_percentages,\n            remove_best=True,\n        )\n        scores[\"method_name\"] = method_name\n        all_scores.append(scores)\n\nscores_df = pd.DataFrame(all_scores)\n</code></pre> <p>     We can clearly see that removing the most valuable data points, as given by the Least Core method, leads to, on average, a decrease in the model's performance and that the method outperforms random removal of data points.    </p> <pre><code>all_scores = []\n\nfor i in trange(5):\n    for method_name in method_names:\n        if method_name == \"Random\":\n            values = ValuationResult.from_random(size=len(utility.data))\n        else:\n            values = compute_least_core_values(\n                u=utility,\n                mode=LeastCoreMode.MonteCarlo,\n                n_iterations=n_iterations,\n                n_jobs=n_jobs,\n            )\n        scores = compute_removal_score(\n            u=utility,\n            values=values,\n            percentages=removal_percentages,\n        )\n        scores[\"method_name\"] = method_name\n        all_scores.append(scores)\n\nscores_df = pd.DataFrame(all_scores)\n</code></pre> <p>     We can clearly see that removing the least valuable data points, as given by the Least Core method, leads to, on average, an increase in the model's performance and that the method outperforms the random removal of data points.    </p>"},{"location":"examples/least_core_basic/#least-core-for-data-valuation","title":"Least Core for Data Valuation","text":"<p>     This notebook introduces Least Core methods for the computation of data values using pyDVL.    </p> <p>     Shapley values define a fair way of distributing the worth of the whole training set when every data point is part of it. But they do not consider the question of stability of subsets: Could some data points obtain a higher payoff if they formed smaller subsets? It is argued that this might be relevant if data providers are paid based on data value, since Shapley values can incentivise them not to contribute their data to the \"grand coalition\", but instead try to form smaller ones. Whether this is of actual practical relevance is debatable, but in any case, the least core is an alternative tool available for any task of Data Valuation    </p> <p>     The Core is another approach to compute data values originating in cooperative game theory that attempts to answer those questions. It is the set of feasible payoffs that cannot be improved upon by a coalition of the participants.    </p> <p>     Its use for Data Valuation was first described in the paper             If You Like Shapley Then You\u2019ll Love the Core            by Tom Yan and Ariel D. Procaccia.    </p> <p>     The Least Core value           \\(v\\)          of the           \\(i\\)          -th sample in dataset           \\(D\\)          wrt. utility           \\(u\\)          is computed by solving the following Linear Program:    </p>      \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{x_i\\in D} v_u(x_i) = u(D) &amp; \\\\ &amp; \\displaystyle\\sum_{x_i\\in S} v_u(x_i) + e \\geq u(S) &amp;, \\forall S \\subset D, S \\neq \\emptyset \\\\ \\end{array} \\]     <p>     To illustrate this method we will use a synthetic dataset. We will first use a subset of 10 data point to compute the exact values and use them to assess the Monte Carlo approximation. Afterwards, we will conduct the data removal experiments as described by Ghorbani and Zou in their paper           Data Shapley: Equitable Valuation of Data for Machine Learning          : We compute the data valuation given different computation budgets and incrementally remove a percentage of the best, respectively worst, data points and observe how that affects the utility.    </p>"},{"location":"examples/least_core_basic/#setup","title":"Setup","text":"<p>     We begin by importing the main libraries and setting some defaults.    </p>      If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/least_core_basic/#dataset","title":"Dataset","text":"<p>     We generate a synthetic dataset using the      <code>       make_classification      </code>      function from scikit-learn.    </p> <p>     We sample 200 data points from a 50-dimensional Gaussian distribution with 25 informative features and 25 non-informative features (generated as random linear combinations of the informative features).    </p> <p>     The 200 samples are uniformly distributed across 3 classes with a small percentage of noise added to the labels to make the task a bit more difficult.    </p>"},{"location":"examples/least_core_basic/#estimating-least-core-values","title":"Estimating Least Core Values","text":"<p>     In this first section we will use a smaller subset of the dataset containing 10 samples in order to be able to compute exact values in a reasonable amount of time. Afterwards, we will use the Monte Carlo method with a limited budget (maximum number of subsets) to approximate these values.    </p>"},{"location":"examples/least_core_basic/#data-removal","title":"Data Removal","text":"<p>     We now move on to the data removal experiments using the full dataset.    </p> <p>     In these experiments, we first rank the data points from most valuable  to least valuable using the values estimated by the Monte Carlo Least Core method. Then, we gradually remove from 5 to 40 percent, by increments of 5 percentage points, of the most valuable/least valuable ones, train the model on this subset and compute its accuracy.    </p>"},{"location":"examples/least_core_basic/#remove-best","title":"Remove Best","text":"<p>     We start by removing the best data points and seeing how the model's accuracy evolves.    </p>"},{"location":"examples/least_core_basic/#remove-worst","title":"Remove Worst","text":"<p>     We then proceed to removing the worst data points and seeing how the model's accuracy evolves.    </p>"},{"location":"examples/msr_banzhaf_digits/","title":"Banzhaf Semivalues","text":"<p>     We will be using the following functions from pyDVL. The main entry point is the function     <code>      compute_banzhaf_semivalues()     </code>     . In order to use it we need the classes           Dataset          ,           Utility          and           Scorer          .    </p> <pre><code>%autoreload\nfrom pydvl.reporting.plots import plot_shapley\nfrom support.banzhaf import load_digits_dataset\nfrom pydvl.value import *\n</code></pre> <pre><code>training_data, _, test_data = load_digits_dataset(\n    test_size=0.3, random_state=random_state\n)\n</code></pre> <p>     Training and test data are then used to instantiate a           Dataset          object:    </p> <pre><code>dataset = Dataset(*training_data, *test_data)\n</code></pre> <pre><code>import torch\nfrom support.banzhaf import TorchCNNModel\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = TorchCNNModel(lr=0.001, epochs=40, batch_size=32, device=device)\nmodel.fit(x=training_data[0], y=training_data[1])\n</code></pre> <pre>\n<code>Train Accuracy: 0.705\nTest Accuracy: 0.630\n</code>\n</pre> <p>     The final component is the scoring function. It can be anything like accuracy or           \\(R^2\\)          , and is set with a string from the           standard sklearn scoring methods          . Please refer to that documentation on information on how to define your own scoring function.    </p> <p>     We group dataset, model and scoring function into an instance of           Utility          and compute the Banzhaf semi-values. We take all defaults, and choose to stop computation using the           MaxChecks          stopping criterion, which terminates after a fixed number of calls to it. With the default     <code>      batch_size     </code>     of 1 this means that we will retrain the model.    </p> <p>     Note how we enable caching using memcached (assuming memcached runs with the default configuration for localhost). This is necessary in the current preliminary implementation of           permutation sampling          , which is the default for           compute_banzhaf_semivalues          .    </p> <pre><code>from pydvl.utils import MemcachedCacheBackend, MemcachedClientConfig\n\n# Compute regular Banzhaf semivalue\nutility = Utility(\n    model=model,\n    data=dataset,\n    scorer=Scorer(\"accuracy\", default=0.0, range=(0, 1)),\n    cache_backend=MemcachedCacheBackend(MemcachedClientConfig()),\n)\nvalues = compute_banzhaf_semivalues(\n    utility, done=MaxChecks(max_checks), n_jobs=n_jobs, progress=True\n)\nvalues.sort(key=\"value\")\ndf = values.to_dataframe(column=\"banzhaf_value\", use_names=True)\n</code></pre> <p>     The returned dataframe contains the mean and variance of the Monte Carlo estimates for the values:    </p>            banzhaf_value                      banzhaf_value_stderr                      banzhaf_value_updates                      156                      -1.097920                      6.662418e-02                      5                      21                      -0.925489                      1.230752e-01                      5                      152                      -0.913313                      3.358054e-02                      5                      73                      -0.778884                      3.668419e-05                      5                      85                      -0.644435                      3.454322e-08                      5           <p>     Let us plot the results. In the next cell we will take the 30 images with the lowest score and plot their values with 95% Normal confidence intervals. Keep in mind that Permutation Monte Carlo Banzhaf is typically very noisy, and it can take many steps to arrive at a clean estimate.    </p> <pre>\n<code>Average value of first 10 data points: 0.650003277874342\nExact values:\n39     0.432836\n45     0.455392\n158    0.533221\n144    0.571260\n36     0.633091\n161    0.697940\n77     0.698507\n28     0.752367\n35     0.838752\n175    0.886668\nName: banzhaf_value, dtype: float64\n</code>\n</pre> <p>     For the first 5 images, we will falsify their label, for images 6-10, we will add some noise.    </p> <pre><code>x_train_anomalous = training_data[0].copy()\ny_train_anomalous = training_data[1].copy()\nanomalous_indices = high_dvl.index.map(int).values[:10]\n\n# Set label of first 5 images to 0\ny_train_anomalous[high_dvl.index.map(int).values[:5]] = 0\n\n# Add noise to images 6-10\nindices = high_dvl.index.values[5:10].astype(int)\ncurrent_images = x_train_anomalous[indices]\nnoisy_images = current_images + 0.5 * np.random.randn(*current_images.shape)\nnoisy_images[noisy_images &amp;lt; 0] = 0.0\nnoisy_images[noisy_images &amp;gt; 1] = 1.0\nx_train_anomalous[indices] = noisy_images\n</code></pre> <pre><code>anomalous_dataset = Dataset(\n    x_train=x_train_anomalous,\n    y_train=y_train_anomalous,\n    x_test=test_data[0],\n    y_test=test_data[1],\n)\n\nanomalous_utility = Utility(\n    model=TorchCNNModel(),\n    data=anomalous_dataset,\n    scorer=Scorer(\"accuracy\", default=0.0, range=(0, 1)),\n    cache_backend=MemcachedCacheBackend(MemcachedClientConfig()),\n)\nanomalous_values = compute_banzhaf_semivalues(\n    anomalous_utility, done=MaxChecks(max_checks), n_jobs=n_jobs, progress=True\n)\nanomalous_values.sort(key=\"value\")\nanomalous_df = anomalous_values.to_dataframe(column=\"banzhaf_value\", use_names=True)\n</code></pre> <p>     Let us now take a look at the low-value images and check how many of our anomalous images are part of it.    </p> <p>     As can be seen in this figure, the valuation of the data points has decreased significantly by adding noise or falsifying their labels. This shows the potential of using Banzhaf values or other data valuation methods to detect mislabeled data points or noisy input data.    </p> <pre>\n<code>Average value of original data points: 0.650003277874342\nAverage value of modified, anomalous data points: -0.02501543656281746\nFor reference, these are the average data values of all data points used for training (anomalous):\nbanzhaf_value            0.006044\nbanzhaf_value_stderr     0.103098\nbanzhaf_value_updates    5.000000\ndtype: float64\nThese are the average data values of all points (original data):\nbanzhaf_value            0.005047\nbanzhaf_value_stderr     0.115262\nbanzhaf_value_updates    5.000000\ndtype: float64\n</code>\n</pre> <pre><code>utility = Utility(\n    model=TorchCNNModel(),\n    data=dataset,\n    scorer=Scorer(\"accuracy\", default=0.0, range=(0, 1)),\n    cache_backend=MemcachedCacheBackend(MemcachedClientConfig()),\n)\n</code></pre> <p>     Computing the values is the same, but we now use a better stopping criterion. Instead of fixing the number of utility evaluations with           MaxChecks          , we use           RankCorrelation          to stop when the change in Spearman correlation between the ranking of two successive iterations is below a threshold.    </p> <pre><code>values = compute_msr_banzhaf_semivalues(\n    utility,\n    done=RankCorrelation(rtol=0.0001, burn_in=10),\n    n_jobs=n_jobs,\n    progress=True,\n)\nvalues.sort(key=\"value\")\nmsr_df = values.to_dataframe(column=\"banzhaf_value\", use_names=True)\n</code></pre> <p>     Inspection of the values reveals (generally) much lower variances. Notice the number of updates to each value as well.    </p>            banzhaf_value                      banzhaf_value_stderr                      banzhaf_value_updates                      137                      -0.264918                      0.093597                      11                      20                      -0.217394                      0.127022                      11                      19                      -0.210309                      0.087179                      11                      41                      -0.210119                      0.071534                      11                      192                      -0.191667                      0.130774                      11           <pre><code>from sklearn.linear_model import SGDClassifier\n\nif is_CI:\n    utility = Utility(\n        model=SGDClassifier(max_iter=2),\n        data=dataset,\n        scorer=Scorer(\"accuracy\", default=0.0, range=(0, 1)),\n    )\nelse:\n    utility = Utility(\n        model=TorchCNNModel(),\n        data=dataset,\n        scorer=Scorer(\"accuracy\", default=0.0, range=(0, 1)),\n    )\n</code></pre> <pre><code>def get_semivalues_and_history(\n    sampler_t, max_checks=max_checks, n_jobs=n_jobs, progress=True\n):\n    _history = HistoryDeviation(n_steps=max_checks, rtol=1e-9)\n    if sampler_t == MSRSampler:\n        semivalue_function = compute_msr_banzhaf_semivalues\n    else:\n        semivalue_function = compute_banzhaf_semivalues\n    _values = semivalue_function(\n        utility,\n        sampler_t=sampler_t,\n        done=MaxChecks(max_checks + 2) | _history,\n        n_jobs=n_jobs,\n        progress=progress,\n    )\n    return _history, _values\n</code></pre> <pre><code># Monte Carlo Permutation Sampling Banzhaf semivalues\nhistory_permutation, permutation_values = get_semivalues_and_history(PermutationSampler)\n</code></pre> <pre><code># MSR Banzhaf values\nhistory_msr, msr_values = get_semivalues_and_history(MSRSampler)\n</code></pre> <pre><code># UniformSampler\nhistory_uniform, uniform_values = get_semivalues_and_history(UniformSampler)\n</code></pre> <pre><code># AntitheticSampler\nhistory_antithetic, antithetic_values = get_semivalues_and_history(AntitheticSampler)\n</code></pre> <pre><code># RandomHierarchicalSampler\nhistory_random, random_values = get_semivalues_and_history(RandomHierarchicalSampler)\n</code></pre> <p>     The plot above visualizes the convergence speed of different samplers used for Banzhaf semivalue calculation. /It shows the average magnitude of how much the semivalues are updated in every step of the algorithm.    </p> <p>     As you can see,             MSR            Banzhaf          stabilizes much faster. After 1000 iterations (subsets sampled and evaluated with the utility), Permutation Monte Carlo Banzhaf has evaluated the marginal function about 5 times per data point (we are using 200 data points). For           MSR          , the semivalue of each data point was updated 1000 times. Due to this, the values converge much faster wrt. the number of utility evaluations, which is the key advantage of           MSR          sampling.    </p> <p>       MSR          sampling does come at a cost, however, which is that the updates to the semivalues are more noisy than in other methods.  We will analyze the impact of this tradeoff in the next sections. First, let us look at how similar all the computed semivalues are. They are all Banzhaf values, so in a perfect world, all samplers should result in the exact same semivalues. However, due to randomness in the utility (recall that we use a neural network) and randomness in the samplers, the resulting values are likely never exactly the same. Another quality measure is that a good sampler would lead to very consistent values, a bad one to less consistent values. Let us first examine how similar the results are, then we'll look at consistency.    </p> <p>     This plot shows that the samplers lead to quite different Banzhaf semivalues, however, all of them have some points in common. The           MSR          Sampler does not seem to be significantly worse than any others.    </p> <p>     In an ideal setting without randomness, the overlap of points would be higher, however, the stochastic nature of the CNN model that we use together with the fact that we use only 200 data points for training, might overshadow these results. As a matter of fact we have the rather discouraging following result:    </p> <pre>\n<code>Total number of top 20 points that all samplers have in common: 0\n</code>\n</pre>"},{"location":"examples/msr_banzhaf_digits/#banzhaf-semi-values-for-data-valuation","title":"Banzhaf Semi-values for data valuation","text":"<p>     This notebook showcases           Data Banzhaf: A Robust Data Valuation Framework for Machine Learning          by Wang, and Jia.    </p> <p>     Computing Banzhaf semi-values using pyDVL follows basically the same procedure as all other semi-value-based methods like Shapley values. However, Data-Banzhaf tends to be more robust to stochasticity in the training process than other semi-values. A property that we study here.    </p> <p>     Additionally, we compare two sampling techniques: the standard permutation-based Monte Carlo sampling, and the so-called           MSR          (Maximum Sample Reuse) principle.    </p> <p>     In order to highlight the strengths of Data-Banzhaf, we require a stochastic model. For this reason, we use a CNN to classify handwritten digits from the           scikit-learn toy datasets          .    </p>"},{"location":"examples/msr_banzhaf_digits/#setup","title":"Setup","text":"If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/msr_banzhaf_digits/#loading-the-dataset","title":"Loading the dataset","text":"<p>     We use a support function,     <code>      load_digits_dataset()     </code>     , which downloads the data and prepares it for usage. It returns four arrays that we then use to construct a           Dataset          . The data consists of grayscale images of shape 8x8 pixels with 16 shades of gray. These images contain handwritten digits from 0 to 9.    </p>"},{"location":"examples/msr_banzhaf_digits/#creating-the-utility-and-computing-banzhaf-semivalues","title":"Creating the utility and computing Banzhaf semivalues","text":"<p>     Now we can calculate the contribution of each training sample to the model performance. First we need a model and a           Scorer          .    </p> <p>     As a model, we use a simple CNN written torch, and wrapped into an object to convert numpy arrays into tensors (as of v0.9.0 valuation methods in pyDVL work only with numpy arrays). Note that any model that implements the protocol           pydvl.utils.types.SupervisedModel          , which is just the standard sklearn interface of     <code>      fit()     </code>     ,     <code>      predict()     </code>     and     <code>      score()     </code>     can be used to construct the utility.    </p>"},{"location":"examples/msr_banzhaf_digits/#evaluation-on-anomalous-data","title":"Evaluation on anomalous data","text":"<p>     An interesting use-case for data valuation is finding anomalous data. Maybe some of the data is really noisy or has been mislabeled. To simulate this, we will change some of the labels of our dataset and add noise to some others. Intuitively, these anomalous data points should then have a lower value.    </p> <p>     To evaluate this, let us first check the average value of the first 10 data points, as these will be the ones that we modify. Currently, these are the 10 data points with the highest values:    </p>"},{"location":"examples/msr_banzhaf_digits/#maximum-sample-reuse-banzhaf","title":"Maximum Sample Reuse Banzhaf","text":"<p>     Despite the previous results already being useful, we had to retrain the model a number of times and yet the variance of the value estimates was high. This has consequences for the stability of the top-k ranking of points, which decreases the applicability of the method. We now introduce a different sampling method called Maximum Sample Reuse (           MSR          ) which reuses every sample for updating the Banzhaf values. The method was introduced by the authors of Data-Banzhaf and is much more sample-efficient, as we will show.    </p> <p>     We next construct a new utility. Note how this time we don't use a cache: the chance of hitting twice the same subset of the training set is low enough that one can dispense with it (nevertheless it can still be useful, e.g. when running many experiments).    </p>"},{"location":"examples/msr_banzhaf_digits/#compare-convergence-speed-of-banzhaf-and-msr-banzhaf-values","title":"Compare convergence speed of Banzhaf and           MSR          Banzhaf Values","text":"<p>     Conventional margin-based samplers produce require evaluating the utility twice to do one update of the value, and permutation samplers do instead           \\(n+1\\)          evaluations for           \\(n\\)          updates. Maximum Sample Reuse (           MSR          ) updates instead all indices in every sample that the utility evaluates. We compare the convergence rates of these methods.    </p> <p>     In order to do so, we will compute the semi-values using different samplers and use a high number of iterations to make sure that the values have converged.    </p>"},{"location":"examples/msr_banzhaf_digits/#similarity-of-the-semivalues-computed-using-different-samplers","title":"Similarity of the semivalues computed using different samplers","text":""},{"location":"examples/msr_banzhaf_digits/#consistency-of-the-semivalues","title":"Consistency of the semivalues","text":"<p>     Finally, we want to analyze how consistent the semivalues returned by the different samplers are. In order to do this, we compute semivalues multiple times and check how many of the data points in the top and lowest 20% of valuation of the data overlap.    </p>"},{"location":"examples/msr_banzhaf_digits/#conclusion","title":"Conclusion","text":"<p>       MSR          sampling updates the semivalue estimates for every index in the sample, much more frequently than any other sampler available, which leads to much           faster convergence          . Additionally, the sampler is more consistent with its value estimates than the other samplers, which might be caused by the higher number of value updates.    </p> <p>     There is alas no general recommendation. It is best to try different samplers when computing semivalues and test which one is best suited for your use case. Nevertheless, the           MSR          sampler seems like a more efficient sampler which may bring fast results and is well-suited for stochastic models.    </p>"},{"location":"examples/shapley_basic_spotify/","title":"Shapley values","text":"<p>     This notebook introduces Shapley methods for the computation of data value using pyDVL.    </p> <p>     In order to illustrate the practical advantages, we will predict the popularity of songs in the dataset           Top Hits Spotify from 2000-2019          , and highlight how data valuation can help investigate and boost the performance of the models. In doing so, we will describe the basic usage patterns of pyDVL.    </p> <p>     Recall that data value is a function of three things:    </p> <ol> <li>      The dataset.     </li> <li>      The model.     </li> <li>      The performance metric or scoring function.     </li> </ol> <p>     Below we will describe how to instantiate each one of these objects and how to use them for data valuation. Please also see the           documentation on data valuation          .    </p> <p>     We will be using the following functions from pyDVL. The main entry point is the function           compute_shapley_values()          , which provides a facade to all Shapley methods. In order to use it we need the classes           Dataset          ,           Utility          and           Scorer          .    </p> <pre><code>%autoreload\nfrom pydvl.reporting.plots import plot_shapley\nfrom pydvl.utils.dataset import GroupedDataset\nfrom support.shapley import load_spotify_dataset\nfrom pydvl.value import *\n</code></pre> <pre><code>training_data, val_data, test_data = load_spotify_dataset(\n    val_size=0.3, test_size=0.3, target_column=\"popularity\", random_state=random_state\n)\n</code></pre> <pre><code>training_data[0].head()\n</code></pre>            artist                      song                      duration_ms                      explicit                      year                      danceability                      energy                      key                      loudness                      mode                      speechiness                      acousticness                      instrumentalness                      liveness                      valence                      tempo                      genre                      1561                      Fetty Wap                      679 (feat. Remy Boyz)                      196693                      True                      2015                      0.618                      0.717                      7                      -5.738                      1                      0.3180                      0.00256                      0.000000                      0.6250                      0.603                      190.050                      8                      1410                      Meghan Trainor                      All About That Bass                      187920                      True                      2015                      0.807                      0.887                      9                      -3.726                      1                      0.0503                      0.05730                      0.000003                      0.1240                      0.961                      134.052                      14                      1772                      Katy Perry                      Chained To The Rhythm                      237733                      False                      2017                      0.562                      0.800                      0                      -5.404                      1                      0.1120                      0.08140                      0.000000                      0.1990                      0.471                      95.029                      14                      1670                      Sigala                      Sweet Lovin' - Radio Edit                      202149                      False                      2015                      0.683                      0.910                      10                      -1.231                      1                      0.0515                      0.05530                      0.000005                      0.3360                      0.674                      124.977                      15                      1780                      Liam Payne                      Strip That Down                      204502                      False                      2017                      0.869                      0.485                      6                      -5.595                      1                      0.0545                      0.24600                      0.000000                      0.0765                      0.527                      106.028                      14           <p>     The dataset has many high-level features, some quite intuitive ('duration_ms' or 'tempo'), while others are a bit more cryptic ('valence'?). For information on each feature, please consult           the dataset's website          .    </p> <p>     In our analysis, we will use all the columns, except for 'artist' and 'song', to predict the 'popularity' of each song. We will nonetheless keep the information on song and artist in a separate object for future reference.    </p> <pre><code>song_name = training_data[0][\"song\"]\nartist = training_data[0][\"artist\"]\ntraining_data[0] = training_data[0].drop([\"song\", \"artist\"], axis=1)\ntest_data[0] = test_data[0].drop([\"song\", \"artist\"], axis=1)\nval_data[0] = val_data[0].drop([\"song\", \"artist\"], axis=1)\n</code></pre> <p>     Input and label data are then used to instantiate a           Dataset          object:    </p> <pre><code>dataset = Dataset(*training_data, *val_data)\n</code></pre> <p>     The calculation of exact Shapley values is computationally very expensive (exponentially so!) because it requires training the model on every possible subset of the training set. For this reason, PyDVL implements techniques to speed up the calculation, such as           Monte Carlo approximations          ,           surrogate models          or           caching          of intermediate results and grouping of data to calculate group Shapley values instead of single data points.    </p> <p>     In our case, we will group songs by artist and calculate the Shapley value for the artists. Given the           pandas Series          for 'artist', to group the dataset by it, one does the following:    </p> <pre><code>grouped_dataset = GroupedDataset.from_dataset(dataset=dataset, data_groups=artist)\n</code></pre> <pre><code>utility = Utility(\n    model=GradientBoostingRegressor(n_estimators=3),\n    data=grouped_dataset,\n    scorer=Scorer(\"neg_mean_absolute_error\", default=0.0),\n)\nvalues = compute_shapley_values(\n    utility,\n    mode=ShapleyMode.TruncatedMontecarlo,\n    # Stop if the standard error is below 1% of the range of the values (which is ~2),\n    # or if the number of updates exceeds 1000\n    done=AbsoluteStandardError(threshold=0.2, fraction=0.9) | MaxUpdates(1000),\n    truncation=RelativeTruncation(utility, rtol=0.01),\n    n_jobs=-1,\n)\nvalues.sort(key=\"value\")\ndf = values.to_dataframe(column=\"data_value\", use_names=True)\n</code></pre> <pre>\n<code>Cancellation of futures is not supported by the joblib backend\n</code>\n</pre> <p>     The function           compute_shapley_values()          serves as a common access point to all Shapley methods. For most of them, we must choose a     <code>      StoppingCriterion     </code>     with the argument     <code>      done=     </code>     . In this case we choose to stop when the ratio of standard error to value is below 0.2 for at least 90% of the training points, or if the number of updates of any index exceeds 1000. The     <code>      mode     </code>     argument specifies the Shapley method to use. In this case, we use the           Truncated Monte Carlo approximation          , which is the fastest of the Monte Carlo methods, owing both to using the permutation definition of Shapley values and the ability to truncate the iteration over a given permutation. We configure this to happen when the contribution of the remaining elements is below 1% of the total utility with the parameter     <code>      truncation=     </code>     and the policy           RelativeTruncation          .    </p> <p>     Let's take a look at the returned dataframe:    </p> <pre><code>df.head()\n</code></pre>            data_value                      data_value_stderr                      Years &amp; Years                      -1.150663                      0.195376                      Reik                      -1.123071                      0.126558                      Astrid S                      -0.945702                      0.331619                      Liam Payne                      -0.886687                      0.112654                      DB Boulevard                      -0.847957                      0.057503           <p>     The first thing to notice is that we sorted the results in ascending order of Shapley value. The index holds the labels for each data group: in this case, artist names. The column     <code>      data_value     </code>     is just that: the Shapley Data value, and     <code>      data_value_stderr     </code>     is its estimated standard error because we are using a Monte Carlo approximation.    </p> <p>     Let us plot the results. In the next cell we will take the 30 artists with the lowest score and plot their values with 95% Normal confidence intervals. Keep in mind that Monte Carlo Shapley is typically very noisy, and it can take many steps to arrive at a clean estimate.    </p> <p>     We can immediately see that many artists (groups of samples) have very low, even negative value, which means that they tend to decrease the total score of the model when present in the training set! What happens if we remove them?    </p> <p>     In the next cell we create a new training set excluding the artists with the lowest scores:    </p> <pre><code>low_dvl_artists = df.iloc[: int(0.2 * len(df))].index.to_list()\nartist_filter = ~artist.isin(low_dvl_artists)\nX_train_good_dvl = training_data[0][artist_filter]\ny_train_good_dvl = training_data[1][artist_filter]\n</code></pre> <p>     Now we will use this \"cleaned\" dataset to retrain the same model and compare its mean absolute error to the one trained on the full dataset. Notice that the score now is calculated using the test set, while in the calculation of the Shapley values we were using the validation set.    </p> <pre><code>model_good_data = GradientBoostingRegressor(n_estimators=3).fit(\n    X_train_good_dvl, y_train_good_dvl\n)\nerror_good_data = mean_absolute_error(\n    model_good_data.predict(test_data[0]), test_data[1]\n)\n\nmodel_all_data = GradientBoostingRegressor(n_estimators=3).fit(\n    training_data[0], training_data[1]\n)\nerror_all_data = mean_absolute_error(model_all_data.predict(test_data[0]), test_data[1])\n\nprint(f\"Improvement: {100*(error_all_data - error_good_data)/error_all_data:02f}%\")\n</code></pre> <pre>\n<code>Improvement: 15.314214%\n</code>\n</pre> <p>     The score has improved by almost 14%! This is quite an important result, as it shows a consistent process to improve the performance of a model by excluding data points from its training set.    </p>      One must however proceed with caution instead of simply throwing away data. For one, `mean_absolute_error` is an estimate of generalization error on unseen data, so the improvement we see on the test set might not be as large upon deployment. It would be advisable to cross-validate this whole process to obtain more conservative estimates. It is also advisable to manually inspect the artists with low value and to try to understand the reason why the model behaves like it does. Finally, remember that **the value depends on the model chosen**! Artists that are detrimental to the Gradient Boosting Regressor might be informative for a different model (although it is likely that the worst ones share some characteristic making them \"bad\" for other regressors).     <p>     Let us take all the songs by Billie Eilish, set their score to 0 and re-calculate the Shapley values.    </p> <pre><code>y_train_anomalous = training_data[1].copy(deep=True)\ny_train_anomalous[artist == \"Billie Eilish\"] = 0\nanomalous_dataset = Dataset(\n    x_train=training_data[0],\n    y_train=y_train_anomalous,\n    x_test=val_data[0],\n    y_test=val_data[1],\n)\ngrouped_anomalous_dataset = GroupedDataset.from_dataset(anomalous_dataset, artist)\nanomalous_utility = Utility(\n    model=GradientBoostingRegressor(n_estimators=3),\n    data=grouped_anomalous_dataset,\n    scorer=Scorer(\"neg_mean_absolute_error\", default=0.0),\n)\nvalues = compute_shapley_values(\n    anomalous_utility,\n    mode=ShapleyMode.TruncatedMontecarlo,\n    done=AbsoluteStandardError(threshold=0.2, fraction=0.9) | MaxUpdates(1000),\n    n_jobs=-1,\n)\nvalues.sort(key=\"value\")\ndf = values.to_dataframe(column=\"data_value\", use_names=True)\n</code></pre> <pre>\n<code>Cancellation of futures is not supported by the joblib backend\n</code>\n</pre> <p>     Let us now consider the low-value artists (at least for predictive purposes, no claims are made about their artistic value!) and plot the results    </p> <p>     And Billie Eilish (our anomalous data group) has moved from top contributor to having negative impact on the performance of the model, as expected!    </p> <p>     What is going on? A popularity of 0 for Billie Eilish's songs is inconsistent with listening patterns for other artists. In artificially setting this, we degrade the predictive power of the model.    </p> <p>     By dropping low-value groups or samples, one can often increase model performance, but by           inspecting          them, it is possible to identify bogus data sources or acquisition methods.    </p>"},{"location":"examples/shapley_basic_spotify/#shapley-for-data-valuation","title":"Shapley for data valuation","text":""},{"location":"examples/shapley_basic_spotify/#setup","title":"Setup","text":"<p>     We begin by importing the main libraries and setting some defaults.    </p>      If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_basic_spotify/#loading-and-grouping-the-dataset","title":"Loading and grouping the dataset","text":"<p>     pyDVL provides a support function for this notebook,     <code>      load_spotify_dataset()     </code>     , which downloads data on songs published after 2014, and splits 30% of data for testing, and 30% of the remaining data for validation. The return value is a triple of training, validation and test data as lists of the form     <code>      [X_input, Y_label]     </code>     .    </p>"},{"location":"examples/shapley_basic_spotify/#creating-the-utility-and-computing-values","title":"Creating the utility and computing values","text":"<p>     Now we can calculate the contribution of each group to the model performance.    </p> <p>     As a model, we use scikit-learn's           GradientBoostingRegressor          , but pyDVL can work with any model from sklearn, xgboost or lightgbm. More precisely, any model that implements the protocol           pydvl.utils.types.SupervisedModel          , which is just the standard sklearn interface of     <code>      fit()     </code>     ,     <code>      predict()     </code>     and     <code>      score()     </code>     can be used to construct the utility.    </p> <p>     The third and final component is the scoring function. It can be anything like accuracy or           \\(R^2\\)          , and is set with a string from the           standard sklearn scoring methods          . Please refer to that documentation on information on how to define your own scoring function.    </p> <p>     We group dataset, model and scoring function into an instance of           Utility          .    </p>"},{"location":"examples/shapley_basic_spotify/#evaluation-on-anomalous-data","title":"Evaluation on anomalous data","text":"<p>     One interesting test is to corrupt some data and to monitor how their value changes. To do this, we will take one of the artists with the highest value and set the popularity of all their songs to 0.    </p>"},{"location":"examples/shapley_knn_flowers/","title":"KNN Shapley","text":"<p>     This notebook shows how to calculate Shapley values for the K-Nearest Neighbours algorithm. By making use of the local structure of KNN, it is possible to compute an exact value in almost linear time, as opposed to exponential complexity of exact, model-agnostic Shapley.    </p> <p>     The main idea is to exploit the fact that adding or removing points beyond the k-ball doesn't influence the score. Because the algorithm then essentially only needs to do a search it runs in           \\(\\mathcal{O}(N \\log N)\\)          time.    </p> <p>     By further using approximate nearest neighbours, it is possible to achieve           \\((\\epsilon,\\delta)\\)          -approximations in sublinear time. However, this is not implemented in pyDVL yet.    </p> <p>     We refer to the original paper that pyDVL implements for details:           Jia, Ruoxi, David Dao, Boxin Wang, Frances Ann Hubis, Nezihe Merve Gurel, Bo Li, Ce Zhang, Costas Spanos, and Dawn Song.             Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms            . Proceedings of the VLDB Endowment 12, no. 11 (1 July 2019): 1610\u201323.      </p> <p>     The main entry point is the function           compute_shapley_values()          , which provides a facade to all Shapley methods. In order to use it we need the classes           Dataset          ,           Utility          and           Scorer          , all of which can be imported from     <code>      pydvl.value     </code>     :    </p> <pre><code>from pydvl.value import *\n</code></pre> <pre><code>sklearn_dataset = datasets.load_iris()\ndata = Dataset.from_sklearn(sklearn_dataset)\nknn = sk.neighbors.KNeighborsClassifier(n_neighbors=5)\nutility = Utility(knn, data)\n</code></pre> <pre><code>shapley_values = compute_shapley_values(utility, mode=ShapleyMode.KNN, progress=True)\nshapley_values.sort(key=\"value\")\nvalues = shapley_values.values\n</code></pre> <pre>\n<code>0it [00:00, ?it/s]</code>\n</pre> <p>     If we now look at  the distribution of Shapley values for each class, we see that each has samples with both high and low scores. This is expected, because an accurate model uses information of all classes.    </p> <pre><code>corrupted_data = deepcopy(data)\nn_corrupted = 10\ncorrupted_data.y_train[:n_corrupted] = (corrupted_data.y_train[:n_corrupted] + 1) % 3\nknn = sk.neighbors.KNeighborsClassifier(n_neighbors=5)\ncontaminated_values = compute_shapley_values(\n    Utility(knn, corrupted_data), mode=ShapleyMode.KNN\n)\n</code></pre> <p>     Taking the average corrupted value and comparing it to non-corrupted ones, we notice that on average anomalous points have a much lower score, i.e. they tend to be much less valuable to the model.    </p> <p>     To do this, first we make sure that we access the results by data index with a call to     <code>      ValuationResult.sort()     </code>     , then we split the values into two groups: corrupted and non-corrupted. Note how we access property     <code>      values     </code>     of the     <code>      ValuationResult     </code>     object. This is a numpy array of values, sorted however the object was sorted. Finally, we compute the quantiles of the two groups and compare them. We see that the corrupted mean is in the lowest percentile of the value distribution, while the correct mean is in the 70th percentile.    </p> <pre><code>contaminated_values.sort(\n    key=\"index\"\n)  # This is redundant, but illustrates sorting, which is in-place\n\ncorrupted_shapley_values = contaminated_values.values[:n_corrupted]\ncorrect_shapley_values = contaminated_values.values[n_corrupted:]\n\nmean_corrupted = np.mean(corrupted_shapley_values)\nmean_correct = np.mean(correct_shapley_values)\npercentile_corrupted = np.round(100 * np.mean(values &amp;lt; mean_corrupted), 0)\npercentile_correct = np.round(100 * np.mean(values &amp;lt; mean_correct), 0)\n\nprint(\n    f\"The corrupted mean is at percentile {percentile_corrupted:.0f} of the value distribution.\"\n)\nprint(\n    f\"The correct mean is percentile {percentile_correct:.0f} of the value distribution.\"\n)\n</code></pre> <pre>\n<code>The corrupted mean is at percentile 2 of the value distribution.\nThe correct mean is percentile 71 of the value distribution.\n</code>\n</pre> <p>     This is confirmed if we plot the distribution of Shapley values and circle corrupt points in red. They all tend to have low Shapley scores, regardless of their position in space and assigned label:    </p>"},{"location":"examples/shapley_knn_flowers/#knn-shapley","title":"KNN Shapley","text":""},{"location":"examples/shapley_knn_flowers/#setup","title":"Setup","text":"<p>     We begin by importing the main libraries and setting some defaults.    </p>      If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_knn_flowers/#building-a-dataset-and-a-utility","title":"Building a Dataset and a Utility","text":"<p>     We use           the sklearn iris dataset          and wrap it into a           pydvl.utils.dataset.Dataset          calling the factory           pydvl.utils.dataset.Dataset.from_sklearn()          . This automatically creates a train/test split for us which will be used to compute the utility.    </p> <p>     We then create a model and instantiate a           Utility          using data and model. The model needs to implement the protocol           pydvl.utils.types.SupervisedModel          , which is just the standard sklearn interface of     <code>      fit()     </code>     ,     <code>      predict()     </code>     and     <code>      score()     </code>     . In constructing the     <code>      Utility     </code>     one can also choose a scoring function, but we pick the default which is just the model's     <code>      knn.score()     </code>     .    </p>"},{"location":"examples/shapley_knn_flowers/#computing-values","title":"Computing values","text":"<p>     Calculating the Shapley values is straightforward. We just call           compute_shapley_values()          with the utility object we created above. The function returns a           ValuationResult          . This object contains the values themselves, data indices and labels.    </p>"},{"location":"examples/shapley_knn_flowers/#inspecting-the-results","title":"Inspecting the results","text":"<p>     Let us first look at the labels' distribution as a function of petal and sepal length:    </p>"},{"location":"examples/shapley_knn_flowers/#corrupting-labels","title":"Corrupting labels","text":"<p>     To test how informative values are, we can corrupt some training labels and see how their Shapley values change with respect to the non-corrupted points.    </p>"},{"location":"examples/shapley_utility_learning/","title":"Data utility learning","text":"<p>     This notebook introduces           Data Utility Learning          , a method of approximating Data Shapley values by learning to estimate the utility function.    </p> <p>     The idea is to employ a model to learn the performance of the learning algorithm of interest on unseen data combinations (i.e. subsets of the dataset). The method was originally described in           Wang, Tianhao, Yu Yang, and Ruoxi Jia.             Improving Cooperative Game Theory-Based Data Valuation via Data Utility Learning            . arXiv, 2022          .    </p> <p>        Warning:            Work on Data Utility Learning is preliminary. It remains to be seen when or whether it can be put effectively into application. For this further testing and benchmarking are required.     </p> <p>     Recall the definition of Shapley value           \\(v_u(i)\\)          for data point           \\(i\\)          :    </p>      \\[\\begin{equation} v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{|S|}^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)] , \\tag{1} \\label{eq:shapley-def} \\end{equation}\\]     <p>     where           \\(N\\)          is the set of all indices in the training set and           \\(u\\)          is the utility.    </p> <p>     In Data Utility Learning, to avoid the exponential cost of computing this sum, one learns a surrogate model for           \\(u\\)          . We start by sampling so-called           utility samples          to form a training set           \\(S_\\mathrm{train}\\)          for our utility model. Each utility sample is a tuple consisting of a subset of indices           \\(S_j\\)          in the dataset and its utility           \\(u(S_j)\\)          :    </p>      \\[\\mathcal{S}_\\mathrm{train} = \\{(S_j, u(S_j): j = 1 , ..., m_\\mathrm{train}\\}\\]     <p>     where           \\(m_\\mathrm{train}\\)          denotes the           training budget          for the learned utility function.    </p> <p>     The subsets are then transformed into boolean vectors           \\(\\phi\\)          in which a           \\(1\\)          at index           \\(k\\)          means that the           \\(k\\)          -th sample of the dataset is present in the subset:    </p>      \\[S_j \\mapsto \\phi_j \\in \\{ 0, 1 \\}^{N}\\]     <p>     We fit a regression model           \\(\\tilde{u}\\)          , called           data utility model          , on the transformed utility samples           \\(\\phi (\\mathcal{S}_\\mathrm{train}) := \\{(\\phi(S_j), u(S_j): j = 1 , ..., m_\\mathrm{train}\\}\\)          and use it to predict instead of computing the utility for any           \\(S_j \\notin \\mathcal{S}_\\mathrm{train}\\)          . We abuse notation and identify           \\(\\tilde{u}\\)          with the composition           \\(\\tilde{u} \\circ \\phi : N \\rightarrow \\mathbb{R}\\)          .    </p> <p>     The main assumption is that it is much faster to fit and use           \\(\\tilde{u}\\)          than it is to compute           \\(u\\)          and that for most           \\(i\\)          ,           \\(v_\\tilde{u}(i) \\approx v_u(i)\\)          in some sense.    </p> <p>     As is the case with all other Shapley methods, the main entry point is the function           compute_shapley_values()          , which provides a facade to all algorithms in this family. We use it with the usual classes           Dataset          and           Utility          . In addition, we must import the core class for learning a utility,           DataUtilityLearning          .    </p> <pre><code>%autoreload\nfrom pydvl.utils import DataUtilityLearning, top_k_value_accuracy\nfrom pydvl.reporting.plots import shaded_mean_std\nfrom pydvl.value import *\n</code></pre> <pre><code>dataset = Dataset.from_sklearn(\n    load_iris(),\n    train_size=train_size,\n    random_state=random_state,\n    stratify_by_target=True,\n)\n</code></pre> <p>     We verify that, as in the paper, if we fit a Support-Vector Classifier to the training data, we obtain an accuracy of around 92%:    </p> <pre><code>model = LinearSVC()\nmodel.fit(dataset.x_train, dataset.y_train)\nprint(f\"Mean accuracy: {100 * model.score(dataset.x_test, dataset.y_test):0.2f}%\")\n</code></pre> <pre>\n<code>Mean accuracy: 92.59%\n</code>\n</pre> <pre><code>computation_times = {}\n</code></pre> <pre><code>utility = Utility(model=model, data=dataset)\n</code></pre> <pre><code>start_time = time.monotonic()\n\nresult = compute_shapley_values(\n    u=utility,\n    mode=ShapleyMode.CombinatorialExact,\n    n_jobs=-1,\n    progress=False,\n)\n\ncomputation_time = time.monotonic() - start_time\ncomputation_times[\"exact\"] = computation_time\n\ndf = result.to_dataframe(column=\"exact\").drop(columns=[\"exact_stderr\"])\n</code></pre> <p>     We now estimate the Data Shapley values using the           DataUtilityLearning          wrapper. This class wraps a           Utility          and delegates calls to it, up until a given budget. Every call yields a utility sample which is saved under the hood for training of the given utility model. Once the budget is exhausted,     <code>      DataUtilityLearning     </code>     fits the model to the utility samples and all subsequent calls use the learned model to predict the wrapped utility instead of delegating to it.    </p> <p>     For the utility model we follow the paper and use a fully connected neural network. To train it we use a total of     <code>      training_budget     </code>     utility samples. We repeat this multiple times for each training budget.    </p>      Note how we use a MonteCarlo approximation instead of `combinatorial_exact` as before. This is because the exact computation samples subsets in a particular order, from the lowest size to the largest. Because the training budget for the model to learn the utility is around 1/4th of the total number of subsets, this would mean that we would never see utility samples for the larger sizes and the model would be biased (try it!)     <pre><code>mlp_kwargs = dict(\n    hidden_layer_sizes=(20, 10),\n    activation=\"relu\",\n    solver=\"adam\",\n    learning_rate_init=0.001,\n    batch_size=batch_size,\n    max_iter=800,\n)\n\nprint(\n    f\"Doing {n_runs} runs for each of {len(training_budget_values)} different training budgets.\"\n)\n\npbar = tqdm(\n    product(range(n_runs), training_budget_values),\n    total=n_runs * len(training_budget_values),\n)\nfor idx, budget in pbar:\n    pbar.set_postfix_str(f\"Run {idx} for training budget: {budget}\")\n    dul_utility = DataUtilityLearning(\n        u=utility, training_budget=budget, model=MLPRegressor(**mlp_kwargs)\n    )\n\n    start_time = time.monotonic()\n\n    # DUL will kick in after training_budget calls to utility\n    result = compute_shapley_values(\n        u=dul_utility,\n        mode=ShapleyMode.PermutationMontecarlo,\n        done=MaxUpdates(300),\n        n_jobs=-1,\n    )\n\n    computation_time = time.monotonic() - start_time\n    if budget in computation_times:\n        computation_times[budget].append(computation_time)\n    else:\n        computation_times[budget] = [computation_time]\n\n    dul_df = result.to_dataframe(column=f\"{budget}_{idx}\").drop(\n        columns=[f\"{budget}_{idx}_stderr\"]\n    )\n    df = pd.concat([df, dul_df], axis=1)\n\ncomputation_times_df = pd.DataFrame(computation_times)\n</code></pre> <pre>\n<code>Doing 10 runs for each of 10 different training budgets.\n</code>\n</pre> <pre>\n<code>  0%|          | 0/100 [00:00&lt;?, ?it/s]</code>\n</pre> <p>     Next we compute the           \\(l_1\\)          error for the different training budgets across all runs and plot mean and standard deviation. We obtain results analogous to Figure 1 of the paper, verifying that the method indeed works for estimating the Data Shapley values (at least in this context).    </p> <p>     In the plot we also display the mean and standard deviation of the computation time taken for each training budget.    </p> <pre><code>errors = np.zeros((len(training_budget_values), n_runs), dtype=float)\naccuracies = np.zeros((len(training_budget_values), n_runs), dtype=float)\n\ntop_k = 3\n\nfor i, budget in enumerate(training_budget_values):\n    for j in range(n_runs):\n        y_true = df[\"exact\"].values\n        y_estimated = df[f\"{budget}_{j}\"].values\n        errors[i, j] = np.linalg.norm(y_true - y_estimated, ord=2)\n        accuracies[i, j] = top_k_value_accuracy(y_true, y_estimated, k=top_k)\n\nerror_from_mean = np.linalg.norm(df[\"exact\"].values - df[\"exact\"].values.mean(), ord=2)\n</code></pre> <p>     Let us next look at how well the ranking of values resulting from using the surrogate           \\(\\tilde{u}\\)          matches the ranking by the exact values. For this we fix           \\(k=3\\)          and consider the           \\(k\\)          samples with the highest value according to           \\(\\tilde{u}\\)          and           \\(u\\)          :    </p> <p>     Finally, for each sample, we look at the distance of the estimates to the exact value across runs. Boxes are centered at the 50th percentile with wiskers at the 25th and 75th. We plot relative distances, as a percentage. We observe a general tendency to underestimate the value:    </p> <pre><code>highest_value_index = df.index[df[\"exact\"].argmax()]\ny_train_corrupted = dataset.y_train.copy()\ny_train_corrupted[highest_value_index] = (\n    y_train_corrupted[highest_value_index] + 1\n) % 3\n\ncorrupted_dataset = Dataset(\n    x_train=dataset.x_train,\n    y_train=y_train_corrupted,\n    x_test=dataset.x_test,\n    y_test=dataset.y_test,\n)\n</code></pre> <p>     We retrain the model on the new dataset and verify that the accuracy decreases:    </p> <pre><code>model = LinearSVC()\nmodel.fit(dataset.x_train, y_train_corrupted)\nprint(f\"Mean accuracy: {100 * model.score(dataset.x_test, dataset.y_test):0.2f}%\")\n</code></pre> <pre>\n<code>Mean accuracy: 82.96%\n</code>\n</pre> <p>     Finally, we recompute the values of all samples using the exact method and the best training budget previously obtained and then plot the resulting scores.    </p> <pre><code>best_training_budget = training_budget_values[errors.mean(axis=1).argmin()]\n\nutility = Utility(\n    model=LinearSVC(),\n    data=corrupted_dataset,\n)\n\nresult = compute_shapley_values(\n    u=utility,\n    mode=ShapleyMode.CombinatorialExact,\n    n_jobs=-1,\n    progress=False,\n)\ndf_corrupted = result.to_dataframe(column=\"exact\").drop(columns=[\"exact_stderr\"])\n\ndul_utility = DataUtilityLearning(\n    u=utility, training_budget=best_training_budget, model=MLPRegressor(**mlp_kwargs)\n)\n\nresult = compute_shapley_values(\n    u=dul_utility,\n    mode=ShapleyMode.PermutationMontecarlo,\n    done=MaxUpdates(300),\n    n_jobs=-1,\n)\ndul_df = result.to_dataframe(column=\"estimated\").drop(columns=[\"estimated_stderr\"])\ndf_corrupted = pd.concat([df_corrupted, dul_df], axis=1)\n</code></pre> <p>     We can see in the figure that both methods assign the lowest value to the sample with the corrupted label.    </p>      As mentioned above, despite the previous results, this work is preliminary and the usefulness of Data Utility Learning remains to be tested in practice."},{"location":"examples/shapley_utility_learning/#data-utility-learning","title":"Data Utility Learning","text":""},{"location":"examples/shapley_utility_learning/#setup","title":"Setup","text":"<p>     We begin by importing the main libraries and setting some defaults.    </p>      If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_utility_learning/#dataset","title":"Dataset","text":"<p>     Following the paper, we take 15 samples (10%) from the           Iris dataset          and compute their Data Shapley values by using all the remaining samples as test set for computing the utility, which in this case is accuracy.    </p>"},{"location":"examples/shapley_utility_learning/#data-shapley","title":"Data Shapley","text":"<p>     We start by defining the utility using the model and computing the exact Data Shapley values by definition           \\(\\ref{eq:shapley-def}\\)          .    </p>"},{"location":"examples/shapley_utility_learning/#evaluation-on-anomalous-data","title":"Evaluation on anomalous data","text":"<p>     One interesting way to assess the Data Utility Learning approach is to corrupt some data and monitor how the value changes. To do this, we will take the sample with the highest score and change its label.    </p>"},{"location":"getting-started/","title":"Getting started","text":"<p>If you want to jump straight in, install pyDVL and then check out the examples. You will probably want to install with support for influence function computation.</p> <p>We have introductions to the ideas behind Data valuation and Influence functions, as well as a short overview of common applications.</p>"},{"location":"getting-started/#","title":"Installing pyDVL","text":"<p>To install the latest release use:</p> <pre><code>pip install pyDVL\n</code></pre> <p>See Extras for optional dependencies, in particular if you are interested in influence functions. You can also install the latest development version from TestPyPI:</p> <pre><code>pip install pyDVL --index-url https://test.pypi.org/simple/\n</code></pre> <p>In order to check the installation you can use:</p> <pre><code>python -c \"import pydvl; print(pydvl.__version__)\"\n</code></pre>"},{"location":"getting-started/#dependencies","title":"Dependencies","text":"<p>pyDVL requires Python &gt;= 3.8, numpy, scikit-learn, scipy, cvxpy for the core methods, and joblib for parallelization locally. Additionally,the Influence functions module requires PyTorch (see Extras below).</p>"},{"location":"getting-started/#installation-extras","title":"Extras","text":"<p>pyDVL has a few extra dependencies that can be optionally installed:</p>"},{"location":"getting-started/#installation-influences","title":"Influence functions","text":"<p>To use the module on influence functions, pydvl.influence, run:</p> <pre><code>pip install pyDVL[influence]\n</code></pre> <p>This includes a dependency on PyTorch (Version 2.0 and above) and thus is left out by default.</p>"},{"location":"getting-started/#cupy","title":"CuPy","text":"<p>In case that you have a supported version of CUDA installed (v11.2 to 11.8 as of this writing), you can enable eigenvalue computations for low-rank approximations with CuPy on the GPU by using:</p> <pre><code>pip install pyDVL[cupy]\n</code></pre> <p>This installs cupy-cuda11x.</p> <p>If you use a different version of CUDA, please install CuPy manually.</p>"},{"location":"getting-started/#ray","title":"Ray","text":"<p>If you want to use Ray to distribute data valuation workloads across nodes in a cluster (it can be used locally as well, but for this we recommend joblib instead) install pyDVL using:</p> <pre><code>pip install pyDVL[ray]\n</code></pre> <p>see the intro to parallelization for more details on how to use it.</p>"},{"location":"getting-started/#memcached","title":"Memcached","text":"<p>If you want to use Memcached for caching utility evaluations, use:</p> <pre><code>pip install pyDVL[memcached]\n</code></pre> <p>This installs pymemcache additionally. Be aware that you still have to start a memcached server manually. See Setting up the Memcached cache.</p>"},{"location":"getting-started/advanced-usage/","title":"Advanced usage","text":"<p>Besides the dos and don'ts of data valuation itself, which are the subject of the examples and the documentation of each method, there are two main things to keep in mind when using pyDVL namely Parallelization and Caching.</p>"},{"location":"getting-started/advanced-usage/#setting-up-parallelization","title":"Parallelization","text":"<p>pyDVL uses parallelization to scale and speed up computations. It does so using one of Dask, Ray or Joblib. The first is used in the influence package whereas the other two are used in the value package.</p>"},{"location":"getting-started/advanced-usage/#data-valuation","title":"Data valuation","text":"<p>For data valuation, pyDVL uses joblib for local parallelization (within one machine) and supports using Ray for distributed parallelization (across multiple machines).</p> <p>The former works out of the box but for the latter you will need to install additional dependencies (see Extras) and to provide a running cluster (or run ray in local mode).</p> <p>Info</p> <p>As of v0.9.0 pyDVL does not allow requesting resources per task sent to the cluster, so you will need to make sure that each worker has enough resources to handle the tasks it receives. A data valuation task using game-theoretic methods will typically make a copy of the whole model and dataset to each worker, even if the re-training only happens on a subset of the data. This means that you should make sure that each worker has enough memory to handle the whole dataset.</p> <p>We use backend classes for both joblib and ray as well as two types of executors for the different algorithms: the first uses a map reduce pattern as seen in the MapReduceJob class and the second implements the futures executor interface from concurrent.futures.</p> <p>As a convenience, you can also instantiate a parallel backend class by using the init_parallel_backend function:</p> <pre><code>from pydvl.parallel import init_parallel_backend\nparallel_backend = init_parallel_backend(backend_name=\"joblib\")\n</code></pre> <p>Info</p> <p>The executor classes are not meant to be instantiated and used by users of pyDVL. They are used internally as part of the computations of the  different methods.</p> <p>Deprecation notice</p> <p>We are currently planning to deprecate MapReduceJob in favour of the futures executor interface because it allows for more diverse computation patterns with interruptions.</p>"},{"location":"getting-started/advanced-usage/#joblib","title":"Joblib","text":"<p>Please follow the instructions in Joblib's documentation for all possible configuration options that you can pass to the parallel_config context manager.</p> <p>To use the joblib parallel backend with the <code>loky</code> backend and verbosity set to <code>100</code> to compute exact shapley values you would use:</p> <pre><code>import joblib\nfrom pydvl.parallel import JoblibParallelBackend\nfrom pydvl.value.shapley import combinatorial_exact_shapley\nfrom pydvl.utils.utility import Utility\n\nparallel_backend = JoblibParallelBackend() \nu = Utility(...)\n\nwith joblib.parallel_config(backend=\"loky\", verbose=100):\n    values = combinatorial_exact_shapley(u, parallel_backend=parallel_backend)\n</code></pre>"},{"location":"getting-started/advanced-usage/#ray","title":"Ray","text":"<p>Additional dependencies</p> <p>The Ray parallel backend requires optional dependencies. See Extras for more information.</p> <p>Please follow the instructions in Ray's documentation to set up a remote cluster. You could alternatively use a local cluster and in that case you don't have to set anything up.</p> <p>Before starting a computation, you should initialize ray by calling  <code>ray.init</code> with the appropriate parameters:</p> <p>To set up and start a local ray cluster with 4 CPUs you would use:</p> <pre><code>import ray\n\nray.init(num_cpus=4)\n</code></pre> <p>Whereas for a remote ray cluster you would use:</p> <pre><code>import ray\n\naddress = \"&lt;Hypothetical Ray Cluster IP Address&gt;\"\nray.init(address)\n</code></pre> <p>To use the ray parallel backend to compute exact shapley values you would use:</p> <pre><code>import ray\nfrom pydvl.parallel import RayParallelBackend\nfrom pydvl.value.shapley import combinatorial_exact_shapley\nfrom pydvl.utils.utility import Utility\n\nray.init()\nparallel_backend = RayParallelBackend()\nu = Utility(...)\nvaues = combinatorial_exact_shapley(u, parallel_backend=parallel_backend)\n</code></pre>"},{"location":"getting-started/advanced-usage/#futures-executor","title":"Futures executor","text":"<p>For the futures executor interface, we have implemented an executor  class for ray in RayExecutor and rely on joblib's loky get_reusable_executor function to instantiate an executor for local parallelization.</p> <p>They are both compatibles with the builtin ThreadPoolExecutor and ProcessPoolExecutor classes.</p> <pre><code>&gt;&gt;&gt; from joblib.externals.loky import _ReusablePoolExecutor\n&gt;&gt;&gt; from pydvl.parallel import JoblibParallelBackend\n&gt;&gt;&gt; parallel_backend = JoblibParallelBackend() \n&gt;&gt;&gt; with parallel_backend.executor() as executor:\n...     results = list(executor.map(lambda x: x + 1, range(3)))\n...\n&gt;&gt;&gt; results\n[1, 2, 3]\n</code></pre>"},{"location":"getting-started/advanced-usage/#map-reduce","title":"Map-reduce","text":"<p>The map-reduce interface is older and more limited in the patterns it allows us to use.</p> <p>To reproduce the previous example using MapReduceJob, we would use:</p> <pre><code>&gt;&gt;&gt; from pydvl.parallel import JoblibParallelBackend, MapReduceJob\n&gt;&gt;&gt; parallel_backend = JoblibParallelBackend() \n&gt;&gt;&gt; map_reduce_job = MapReduceJob(\n...     list(range(3)),\n...     map_func=lambda x: x[0] + 1,\n...     parallel_backend=parallel_backend,\n... )\n&gt;&gt;&gt; results = map_reduce_job()\n&gt;&gt;&gt; results\n[1, 2, 3]\n</code></pre>"},{"location":"getting-started/advanced-usage/#influence-functions","title":"Influence functions","text":"<p>Refer to Scaling influence computation for explanations about parallelization for Influence Functions.</p>"},{"location":"getting-started/advanced-usage/#getting-started-cache","title":"Caching","text":"<p>PyDVL can cache (memoize) the computation of the utility function and speed up some computations for data valuation. It is however disabled by default. When it is enabled it takes into account the data indices passed as argument and the utility function wrapped into the Utility object. This means that care must be taken when reusing the same utility function with different data, see the documentation for the caching package for more information.</p> <p>In general, caching won't play a major role in the computation of Shapley values because the probability of sampling the same subset twice, and hence needing the same utility function computation, is very low. However, it can be very useful when comparing methods that use the same utility function, or when running multiple experiments with the same data.</p> <p>pyDVL supports 3 different caching backends:</p> <ul> <li> <p>InMemoryCacheBackend:   an in-memory cache backend that uses a dictionary to store and retrieve   cached values. This is used to share cached values between threads   in a single process.</p> </li> <li> <p>DiskCacheBackend:   a disk-based cache backend that uses pickled values written to and read from disk.   This is used to share cached values between processes in a single machine.</p> </li> <li> <p>MemcachedCacheBackend:   a Memcached-based cache backend that uses pickled values written to   and read from a Memcached server. This is used to share cached values   between processes across multiple machines.</p> Memcached extras <p>The Memcached backend requires optional dependencies.  See Extras for more information.</p> </li> </ul> <p>As an example, here's how one would use the disk-based cached backend with a utility:</p> <pre><code>from pydvl.utils.caching.disk import DiskCacheBackend\nfrom pydvl.utils.utility import Utility\n\ncache_backend = DiskCacheBackend()\nu = Utility(..., cache_backend=cache_backend)\n</code></pre> <p>Please refer to the documentation and examples of each backend class for more details.</p> <p>When is the cache really necessary?</p> <p>Crucially, semi-value computations with the PermutationSampler require caching to be enabled, or they will take twice as long as the direct implementation in compute_shapley_values.</p> <p>Using the cache</p> <p>Continue reading about the cache in the documentation for the caching package.</p>"},{"location":"getting-started/advanced-usage/#setting-up-memcached","title":"Setting up the Memcached cache","text":"<p>Memcached is an in-memory key-value store accessible over the network. pyDVL can use it to cache the computation of the utility function and speed up some computations (in particular, semi-value computations with the PermutationSampler but other methods may benefit as well).</p> <p>You can either install it as a package or run it inside a docker container (the simplest). For installation instructions, refer to the Getting started section in memcached's wiki. Then you can run it with:</p> <pre><code>memcached -u user\n</code></pre> <p>To run memcached inside a container in daemon mode instead, use:</p> <pre><code>docker container run -d --rm -p 11211:11211 memcached:latest\n</code></pre>"},{"location":"getting-started/applications/","title":"Applications of data valuation","text":"<p>Data valuation methods can improve various aspects of data engineering and machine learning workflows. When applied judiciously, these methods can enhance data quality, model performance, and cost-effectiveness.</p> <p>However, the results can be inconsistent. Values have a strong dependency on the training procedure and the performance metric used. For instance, accuracy is a poor metric for imbalanced sets and this has a stark effect on data values. Some models exhibit great variance in some regimes and this again has a detrimental effect on values. See Problems of data values for more on this.</p> <p>Here we quickly enumerate the most common uses of data valuation. For a comprehensive overview, along with concrete examples, please refer to the Transferlab blog post on this topic.</p>"},{"location":"getting-started/applications/#data-engineering","title":"Data engineering","text":"<p>Some of the promising applications in data engineering include:</p> <ul> <li>Removing low-value data points to increase model performance.</li> <li>Pruning redundant samples enables more efficient training of large models.</li> <li>Active learning. Points predicted to have high-value points can be prioritized   for labeling, reducing the cost of data collection.</li> <li>Analyzing high- and low-value data to guide data collection and improve   upstream data processes. Low-value points may reveal data issues to address.</li> <li>Identify irrelevant or duplicated data when evaluating offerings from data   providers.</li> </ul>"},{"location":"getting-started/applications/#model-development","title":"Model development","text":"<p>Some of the useful applications include:</p> <ul> <li>Data attribution for interpretation and debugging: Analyzing the most or least   valuable samples for a class can reveal cases where the model relies on   confounding features instead of true signal. Investigating influential points   for misclassified examples highlights limitations to address.</li> <li>Sensitivity / robustness analysis: (Broderick et al., 2021)<sup>1</sup> shows that   removing a small fraction of highly influential data can completely flip model   conclusions. This can reveal potential issues with the modeling approach, data   collection process, or intrinsic difficulties of the problem that require   further inspection.</li> <li>Continual learning: in order to avoid forgetting when training on new data,   a subset of previously seen data is presented again. Data valuation can help   in the selection of the most valuable samples to retain.</li> </ul>"},{"location":"getting-started/applications/#attacks","title":"Attacks","text":"<p>Data valuation techniques have applications in detecting data manipulation and contamination, although the feasibility of such attacks is limited.</p> <ul> <li>Watermark removal: Points with low value on a correct validation set may be   part of a watermarking mechanism.</li> <li>Poisoning attacks: Influential points can be shifted to induce large changes   in model estimators.</li> </ul>"},{"location":"getting-started/applications/#data-markets","title":"Data markets","text":"<p>Additionally, one of the motivating applications for the whole field is that of data markets, where data valuation can be the key component to determine the price of data.</p> <p>Game-theoretic valuation methods like Shapley values can help assign fair prices, but have limitations around handling duplicates or adversarial data. Model-free methods like LAVA (Just et al., 2023)<sup>2</sup> and CRAIG are particularly well suited for this, as they use the Wasserstein distance between a vendor's data and the buyer's to determine the value of the former. </p> <p>However, this is a complex problem which can face simple practical problems like data owners not willing to disclose their data for valuation, even to a broker.</p> <ol> <li> <p>Broderick, T., Giordano, R., Meager, R., 2021. An Automatic Finite-Sample Robustness Metric: When Can Dropping a Little Data Make a Big Difference? \u21a9</p> </li> <li> <p>Just, H.A., Kang, F., Wang, T., Zeng, Y., Ko, M., Jin, M., Jia, R., 2023. LAVA: Data Valuation without Pre-Specified Learning Algorithms. Presented at the The Eleventh International Conference on Learning Representations (ICLR 2023).\u00a0\u21a9</p> </li> </ol>"},{"location":"getting-started/benchmarking/","title":"Benchmarking tasks","text":"<p>Because the magnitudes of values or influences from different algorithms, or datasets, are not comparable to each other, evaluation of the methods is typically done with downstream tasks.</p>"},{"location":"getting-started/benchmarking/#benchmarking-valuation-methods","title":"Benchmarking valuation methods","text":"<p>Data valuation is particularly useful for data selection, pruning and inspection in general. For this reason, the most common benchmarks are data removal and noisy label detection.</p>"},{"location":"getting-started/benchmarking/#high-value-point-removal","title":"High-value point removal","text":"<p>After computing the values for all data in \\(T = \\{ \\mathbf{z}_i : i = 1, \\ldots, n \\}\\), the set is sorted by decreasing value. We denote by \\(T_{[i :]}\\) the sorted sequence of points \\((\\mathbf{z}_i, \\mathbf{z}_{i + 1}, \\ldots, \\mathbf{z}_n)\\) for \\(1 \\leqslant i \\leqslant n\\). Now train successively \\(f_{T [i :]}\\) and compute its accuracy \\(a_{T_{[i :]}} (D_{\\operatorname{test}})\\) on the held-out test set, then plot all numbers. By using \\(D_{\\operatorname{test}}\\) one approximates the expected accuracy drop on unseen data. Because the points removed have a high value, one expects performance to drop visibly wrt. a random baseline.</p>"},{"location":"getting-started/benchmarking/#low-value-point-removal","title":"Low-value point removal","text":"<p>The complementary experiment removes data in increasing order, with the lowest valued points first. Here one expects performance to increase relatively to randomly removing points before training. Additionally, every real dataset will include slightly out-of-distribution points, so one should also expect an absolute increase in performance when some of the lowest valued points are removed.</p>"},{"location":"getting-started/benchmarking/#value-transfer","title":"Value transfer","text":"<p>This experiment explores the extent to which data values computed with one (cheap) model can be transferred to another (potentially more complex) one. Different classifiers are used as a source to calculate data values. These values are then used in the point removal tasks described above, but using a different (target) model for evaluation of the accuracies \\(a_{T [i :]}\\). A multi-layer perceptron is added for evaluation as well.</p>"},{"location":"getting-started/benchmarking/#noisy-label-detection","title":"Noisy label detection","text":"<p>This experiment tests the ability of a method to detect mislabeled instances in the data. A fixed fraction \\(\\alpha\\) of the training data are picked at random and their labels flipped. Data values are computed, then the \\(\\alpha\\)-fraction of lowest-valued points are selected, and the overlap with the subset of flipped points is computed. This synthetic experiment is however hard to put into practical use, since the fraction \\(\\alpha\\) is of course unknown in practice.</p>"},{"location":"getting-started/benchmarking/#rank-stability","title":"Rank stability","text":"<p>Introduced in [@wang_data_2022], one can  look at how stable the top \\(k\\)% of the values is across runs. Rank stability of a method is necessary but not sufficient for good results. Ideally one wants to identify high-value points reliably (good precision and recall) and consistently (good rank stability).</p>"},{"location":"getting-started/benchmarking/#benchmarking-influence-function-methods","title":"Benchmarking Influence function methods","text":"<p>Todo</p> <p>This section is basically a stub</p> <p>Although in principle one can compute the average influence over the test set and run the same tasks as above, because influences are computed for each pair of training and test sample, they typically require different experiments to compare their efficacy.</p>"},{"location":"getting-started/benchmarking/#approximation-quality","title":"Approximation quality","text":"<p>The biggest difficulty when computing influences is the approximation of the inverse Hessian-vector product. For this reason one often sees in the literature the quality of the approximation to LOO as an indicator of performance, the exact Influence Function being a first order approximation to it. However, as shown by (Bae et al., 2022)<sup>1</sup>, the different approximation errors ensuing for lack of convexity, approximate Hessian-vector products and so on, lead to this being a poor benchmark overall.</p>"},{"location":"getting-started/benchmarking/#data-re-labelling","title":"Data re-labelling","text":"<p>(Kong et al., 2022)<sup>2</sup> introduce a method using IFs to re-label harmful training samples in order to improve accuracy. One can then take the obtained improvement as a measure of the quality of the IF method.</p>"},{"location":"getting-started/benchmarking/#post-hoc-fairness-adjustment","title":"Post-hoc fairness adjustment","text":"<p>Introduced in [@...], the idea is to compute influences over a carefully selected fair set, and using them to re-weight the training data.</p> <ol> <li> <p>Bae, J., Ng, N., Lo, A., Ghassemi, M., Grosse, R.B., 2022. If Influence Functions are the Answer, Then What is the Question?, in: Advances in Neural Information Processing Systems. Presented at the NeurIPS 2022, pp. 17953\u201317967.\u00a0\u21a9</p> </li> <li> <p>Kong, S., Shen, Y., Huang, L., 2022. Resolving Training Biases via Influence-based Data Relabeling. Presented at the International Conference on Learning Representations (ICLR 2022).\u00a0\u21a9</p> </li> </ol>"},{"location":"getting-started/first-steps/","title":"First steps","text":"<p>Warning</p> <p>Make sure you have read Getting started before using the library.  In particular read about which extra dependencies you may need.</p>"},{"location":"getting-started/first-steps/#main-concepts","title":"Main concepts","text":"<p>pyDVL aims to be a repository of production-ready, reference implementations of algorithms for data valuation and influence functions. Even though we only briefly introduce key concepts in the documentation, the following sections  should be enough to get you started.</p> <ul> <li>Basics of data valuation for key objects and usage patterns for Shapley value   computation and related methods.</li> <li>Computing Influence Values for instructions on how to compute influence functions.</li> </ul>"},{"location":"getting-started/first-steps/#running-the-examples","title":"Running the examples","text":"<p>If you are somewhat familiar with the concepts of data valuation, you can start by browsing our worked-out examples illustrating pyDVL's capabilities either:</p> <ul> <li>In the examples under Basics of data valuation and Computing Influence Values.</li> <li>Using binder notebooks, deployed from each   example's page.</li> <li>Locally, by starting a jupyter server at the root of the project. You will   have to install jupyter first manually since it's not a dependency of the   library.</li> </ul>"},{"location":"getting-started/first-steps/#advanced-usage","title":"Advanced usage","text":"<p>Refer to the Advanced usage page for explanations on how to enable and use parallelization and caching.</p>"},{"location":"getting-started/glossary/","title":"Glossary","text":"<p>This glossary is meant to provide only brief explanations of each term, helping to clarify the concepts and techniques used in the library. For more detailed information, please refer to the relevant literature or resources.</p> <p>Warning</p> <p>This glossary is still a work in progress. Pull requests are welcome!</p> <p>Terms in data valuation and influence functions:</p>"},{"location":"getting-started/glossary/#arnoldi-method","title":"Arnoldi Method","text":"<p>The Arnoldi method approximately computes eigenvalue, eigenvector pairs of a symmetric matrix. For influence functions, it is used to approximate the iHVP. Introduced by (Schioppa et al., 2022)<sup>1</sup> in the context of influence functions.</p> <ul> <li>Implementation (torch)     </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#block-conjugate-gradient","title":"Block Conjugate Gradient","text":"<p>A blocked version of CG, which solves several linear systems simultaneously. For Influence Functions, it is used to approximate the iHVP.</p> <ul> <li>Implementation (torch)    </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#class-wise-shapley","title":"Class-wise Shapley","text":"<p>Class-wise Shapley is a Shapley valuation method which introduces a utility function that balances in-class, and out-of-class accuracy, with the goal of favoring points that improve the model's performance on the class they belong to. It is estimated to be particularly useful in imbalanced datasets, but more research is needed to confirm this. Introduced by (Schoch et al., 2022)<sup>2</sup>.</p> <ul> <li>Implementation    </li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#conjugate-gradient","title":"Conjugate Gradient","text":"<p>CG is an algorithm for solving linear systems with a symmetric and positive-definite coefficient matrix. For Influence Functions, it is used to approximate the iHVP.</p> <ul> <li>Implementation (torch) </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#data-utility-learning","title":"Data Utility Learning","text":"<p>Data Utility Learning is a method that uses an ML model to learn the utility function. Essentially, it learns to predict the performance of a model when trained on a given set of indices from the dataset. The cost of training this model is quickly amortized by avoiding costly re-evaluations of the original utility. Introduced by (Wang et al., 2022)<sup>3</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#eigenvalue-corrected-kronecker-factored-approximate-curvature","title":"Eigenvalue-corrected Kronecker-Factored Approximate Curvature","text":"<p>EKFAC builds on K-FAC by correcting for the approximation errors in the eigenvalues of the blocks of the Kronecker-factored approximate curvature matrix. This correction aims to refine the accuracy of natural gradient approximations, thus potentially offering better training efficiency and stability in neural networks.</p> <ul> <li>Implementation (torch)    </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#group-testing","title":"Group Testing","text":"<p>Group Testing is a strategy for identifying characteristics within groups of items efficiently, by testing groups rather than individuals to quickly narrow down the search for items with specific properties. Introduced into data valuation by (Jia et al., 2019)<sup>4</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#influence-function","title":"Influence Function","text":"<p>The Influence Function measures the impact of a single data point on a statistical estimator. In machine learning, it's used to understand how much a particular data point affects the model's prediction. Introduced into data valuation by (Koh and Liang, 2017)<sup>5</sup>.</p> <ul> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#inverse-hessian-vector-product","title":"Inverse Hessian-vector product","text":"<p>iHVP is the operation of calculating the product of the inverse Hessian matrix of a function and a vector, without explicitly constructing nor inverting the full Hessian matrix first. This is essential for influence function computation.</p>"},{"location":"getting-started/glossary/#kronecker-factored-approximate-curvature","title":"Kronecker-Factored Approximate Curvature","text":"<p>K-FAC is an optimization technique that approximates the Fisher Information matrix's inverse efficiently. It uses the Kronecker product to factor the matrix, significantly speeding up the computation of natural gradient updates and potentially improving training efficiency.</p>"},{"location":"getting-started/glossary/#least-core","title":"Least Core","text":"<p>The Least Core is a solution concept in cooperative game theory, referring to the smallest set of payoffs to players that cannot be improved upon by any coalition, ensuring stability in the allocation of value. In data valuation, it implies solving a linear and a quadratic system whose constraints are determined by the evaluations of the utility function on every subset of the training data. Introduced as data valuation method by (Yan and Procaccia, 2021)<sup>6</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#linear-time-stochastic-second-order-algorithm","title":"Linear-time Stochastic Second-order Algorithm","text":"<p>LiSSA is an efficient algorithm for approximating the inverse Hessian-vector product, enabling faster computations in large-scale machine learning problems, particularly for second-order optimization. For Influence Functions, it is used to approximate the iHVP. Introduced by (Agarwal et al., 2017)<sup>7</sup>.</p> <ul> <li>Implementation (torch)    </li> <li>Documentation (torch)    </li> </ul>"},{"location":"getting-started/glossary/#leave-one-out","title":"Leave-One-Out","text":"<p>LOO in the context of data valuation refers to the process of evaluating the impact of removing individual data points on the model's performance. The value of a training point is defined as the marginal change in the model's performance when that point is removed from the training set.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#maximum-sample-reuse","title":"Maximum Sample Reuse","text":"<p>MSR is a sampling method for data valuation that updates the value of every data point in one sample. This method can achieve much faster convergence. Introduced by (Wang and Jia, 2023)<sup>8</sup></p> <ul> <li>Implementation</li> </ul>"},{"location":"getting-started/glossary/#monte-carlo-least-core","title":"Monte Carlo Least Core","text":"<p>MCLC is a variation of the Least Core that uses a reduced amount of constraints, sampled randomly from the powerset of the training data. Introduced by (Yan and Procaccia, 2021)<sup>6</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#monte-carlo-shapley","title":"Monte Carlo Shapley","text":"<p>MCS estimates the Shapley Value using a Monte Carlo approximation to the sum over subsets of the training set. This reduces computation to polynomial time at the cost of accuracy, but this loss is typically irrelevant for downstream applications in ML. Introduced into data valuation by (Ghorbani and Zou, 2019)<sup>9</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#nystrom-low-rank-approximation","title":"Nystr\u00f6m Low-Rank Approximation","text":"<p>The Nystr\u00f6m approximation computes a low-rank approximation to a symmetric positive-definite matrix via random projections. For influence functions,  it is used to approximate the iHVP. Introduced as sketch and solve algorithm in (Hataya and Yamada, 2023)<sup>10</sup>, and as preconditioner for PCG in  (Frangella et al., 2023)<sup>11</sup>.</p> <ul> <li>Implementation Sketch-and-Solve (torch)    </li> <li>Documentation Sketch-and-Solve (torch)</li> <li>Implementation Preconditioner (torch)    </li> </ul>"},{"location":"getting-started/glossary/#point-removal-task","title":"Point removal task","text":"<p>A task in data valuation where the quality of a valuation method is measured through the impact of incrementally removing data points on the model's performance, where the points are removed in order of their value. See</p> <ul> <li>Benchmarking tasks</li> </ul>"},{"location":"getting-started/glossary/#preconditioned-block-conjugate-gradient","title":"Preconditioned Block Conjugate Gradient","text":"<p>A blocked version of PCG, which solves  several linear systems simultaneously. For Influence Functions, it is used to approximate the iHVP.</p> <ul> <li>Implementation CG (torch)    </li> <li>Implementation Preconditioner (torch)    </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#preconditioned-conjugate-gradient","title":"Preconditioned Conjugate Gradient","text":"<p>A preconditioned version of CG for improved convergence, depending on the characteristics of the matrix and the preconditioner. For Influence Functions, it is used to approximate the iHVP.</p> <ul> <li>Implementation CG (torch)    </li> <li>Implementation Preconditioner (torch)    </li> <li>Documentation (torch)</li> </ul>"},{"location":"getting-started/glossary/#shapley-value","title":"Shapley Value","text":"<p>Shapley Value is a concept from cooperative game theory that allocates payouts to players based on their contribution to the total payoff. In data valuation, players are data points. The method assigns a value to each data point based on a weighted average of its marginal contributions to the model's performance  when trained on each subset of the training set. This requires \\(\\mathcal{O}(2^{n-1})\\) re-trainings of the model, which is infeasible for even trivial data set sizes, so one resorts to approximations like TMCS. Introduced into data valuation by (Ghorbani and Zou, 2019)<sup>9</sup>.</p> <ul> <li>Implementation</li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#truncated-monte-carlo-shapley","title":"Truncated Monte Carlo Shapley","text":"<p>TMCS is an efficient approach to estimating the Shapley Value using a truncated version of the Monte Carlo method, reducing computation time while maintaining accuracy in large datasets. Introduced by (Ghorbani and Zou, 2019)<sup>9</sup>.</p> <ul> <li>Implementation    </li> <li>Documentation</li> </ul>"},{"location":"getting-started/glossary/#weighted-accuracy-drop","title":"Weighted Accuracy Drop","text":"<p>WAD is a metric to evaluate the impact of sequentially removing data points on the performance of a machine learning model, weighted by their rank, i.e. by the time at which they were removed. Introduced by (Schoch et al., 2022)<sup>2</sup>.</p>"},{"location":"getting-started/glossary/#other-terms","title":"Other terms","text":""},{"location":"getting-started/glossary/#coefficient-of-variation","title":"Coefficient of Variation","text":"<p>CV is a statistical measure of the dispersion of data points in a data series around the mean, expressed as a percentage. It's used to compare the degree of variation from one data series to another, even if the means are drastically different.</p>"},{"location":"getting-started/glossary/#constraint-satisfaction-problem","title":"Constraint Satisfaction Problem","text":"<p>A CSP involves finding values for variables within specified constraints or conditions, commonly used in scheduling, planning, and design problems where solutions must satisfy a set of restrictions.</p>"},{"location":"getting-started/glossary/#out-of-bag","title":"Out-of-Bag","text":"<p>OOB refers to data samples in an ensemble learning context (like random forests) that are not selected for training a specific model within the ensemble. These OOB samples are used as a validation set to estimate the model's accuracy, providing a convenient internal cross-validation mechanism.</p>"},{"location":"getting-started/glossary/#machine-learning-reproducibility-challenge","title":"Machine Learning Reproducibility Challenge","text":"<p>The MLRC is an initiative that encourages the verification and replication of machine learning research findings, promoting transparency and reliability in the field. Papers are published in Transactions on Machine Learning Research (TMLR).</p> <ol> <li> <p>Schioppa, A., Zablotskaia, P., Vilar, D., Sokolov, A., 2022. Scaling Up Influence Functions. Proc. AAAI Conf. Artif. Intell. 36, 8179\u20138186. https://doi.org/10.1609/aaai.v36i8.20791 \u21a9</p> </li> <li> <p>Schoch, S., Xu, H., Ji, Y., 2022. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification, in: Proc. Of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). Presented at the Advances in Neural Information Processing Systems (NeurIPS 2022).\u00a0\u21a9\u21a9</p> </li> <li> <p>Wang, T., Yang, Y., Jia, R., 2022. Improving Cooperative Game Theory-based Data Valuation via Data Utility Learning. Presented at the International Conference on Learning Representations (ICLR 2022). Workshop on Socially Responsible Machine Learning, arXiv. https://doi.org/10.48550/arXiv.2107.06336 \u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Gurel, N.M., Li, B., Zhang, C., Spanos, C., Song, D., 2019. Efficient task-specific data valuation for nearest neighbor algorithms. Proc. VLDB Endow. 12, 1610\u20131623. https://doi.org/10.14778/3342263.3342637 \u21a9</p> </li> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions, in: Proceedings of the 34th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 1885\u20131894.\u00a0\u21a9</p> </li> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9\u21a9</p> </li> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. JMLR 18, 1\u201340.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T., Jia, R., 2023. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning, in: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 6388\u20136421.\u00a0\u21a9</p> </li> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9\u21a9\u21a9</p> </li> <li> <p>Hataya, R., Yamada, M., 2023. Nystr\u00f6m Method for Accurate and Scalable Implicit Differentiation, in: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 4643\u20134654.\u00a0\u21a9</p> </li> <li> <p>Frangella, Z., Tropp, J.A., Udell, M., 2023. Randomized Nystr\u00f6m Preconditioning. SIAM J. Matrix Anal. Appl. 44, 718\u2013752. https://doi.org/10.1137/21M1466244 \u21a9</p> </li> </ol>"},{"location":"getting-started/methods/","title":"Methods","text":"<p>We currently implement the following methods:</p>"},{"location":"getting-started/methods/#data-valuation","title":"Data valuation","text":"<ul> <li> <p>LOO.</p> </li> <li> <p>Permutation Shapley   (also called ApproxShapley) (Castro et al., 2009)<sup>1</sup>.</p> </li> <li> <p>TMCS   (Ghorbani and Zou, 2019)<sup>2</sup>.</p> </li> <li> <p>Data Banzhaf   [@wang_data_2022].</p> </li> <li> <p>Beta Shapley   (Kwon and Zou, 2022)<sup>3</sup>.</p> </li> <li> <p>CS-Shapley   (Schoch et al., 2022)<sup>4</sup>.</p> </li> <li> <p>Least Core   (Yan and Procaccia, 2021)<sup>5</sup>.</p> </li> <li> <p>Owen Sampling   (Okhrati and Lipani, 2021)<sup>6</sup>.</p> </li> <li> <p>Data Utility Learning   (Wang et al., 2022)<sup>7</sup>.</p> </li> <li> <p>kNN-Shapley   (Jia et al., 2019)<sup>8</sup>.</p> </li> <li> <p>Group Testing   (Jia et al., 2019)<sup>9</sup></p> </li> <li> <p>Data-OOB   (Kwon and Zou, 2023)<sup>10</sup>.</p> </li> </ul>"},{"location":"getting-started/methods/#influence-functions","title":"Influence functions","text":"<ul> <li> <p>CG Influence.   (Koh and Liang, 2017)<sup>11</sup>.</p> </li> <li> <p>Direct Influence   (Koh and Liang, 2017)<sup>11</sup>.</p> </li> <li> <p>LiSSA   (Agarwal et al., 2017)<sup>12</sup>.</p> </li> <li> <p>Arnoldi Influence   (Schioppa et al., 2022)<sup>13</sup>.</p> </li> <li> <p>EKFAC Influence   (George et al., 2018; Martens and Grosse, 2015)<sup>14</sup><sup>15</sup>.</p> </li> <li> <p>Nystr\u00f6m Influence, based   on the ideas in (Hataya and Yamada, 2023)<sup>16</sup> for bi-level optimization.</p> </li> </ul> <ol> <li> <p>Castro, J., G\u00f3mez, D., Tejada, J., 2009. Polynomial calculation of the Shapley value based on sampling. Computers   &amp; Operations Research, Selected papers presented at the Tenth International Symposium on Locational Decisions (ISOLDE X) 36, 1726\u20131730. https://doi.org/10.1016/j.cor.2008.04.004 \u21a9</p> </li> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning, in: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022,. Presented at the AISTATS 2022, PMLR.\u00a0\u21a9</p> </li> <li> <p>Schoch, S., Xu, H., Ji, Y., 2022. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification, in: Proc. Of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). Presented at the Advances in Neural Information Processing Systems (NeurIPS 2022).\u00a0\u21a9</p> </li> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9</p> </li> <li> <p>Okhrati, R., Lipani, A., 2021. A Multilinear Sampling Algorithm to Estimate Shapley Values, in: 2020 25th International Conference on Pattern Recognition (ICPR). Presented at the 2020 25th International Conference on Pattern Recognition (ICPR), IEEE, pp. 7992\u20137999. https://doi.org/10.1109/ICPR48806.2021.9412511 \u21a9</p> </li> <li> <p>Wang, T., Yang, Y., Jia, R., 2022. Improving Cooperative Game Theory-based Data Valuation via Data Utility Learning. Presented at the International Conference on Learning Representations (ICLR 2022). Workshop on Socially Responsible Machine Learning, arXiv. https://doi.org/10.48550/arXiv.2107.06336 \u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Gurel, N.M., Li, B., Zhang, C., Spanos, C., Song, D., 2019. Efficient task-specific data valuation for nearest neighbor algorithms. Proc. VLDB Endow. 12, 1610\u20131623. https://doi.org/10.14778/3342263.3342637 \u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Hynes, N., G\u00fcrel, N.M., Li, B., Zhang, C., Song, D., Spanos, C.J., 2019. Towards Efficient Data Valuation Based on the Shapley Value, in: Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics (AISTATS), PMLR, pp. 1167\u20131176.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2023. Data-OOB: Out-of-bag Estimate as a Simple and Efficient Data Value, in: Proceedings of the 40th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 18135\u201318152.\u00a0\u21a9</p> </li> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions, in: Proceedings of the 34th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 1885\u20131894.\u00a0\u21a9\u21a9</p> </li> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. JMLR 18, 1\u201340.\u00a0\u21a9</p> </li> <li> <p>Schioppa, A., Zablotskaia, P., Vilar, D., Sokolov, A., 2022. Scaling Up Influence Functions. Proc. AAAI Conf. Artif. Intell. 36, 8179\u20138186. https://doi.org/10.1609/aaai.v36i8.20791 \u21a9</p> </li> <li> <p>George, T., Laurent, C., Bouthillier, X., Ballas, N., Vincent, P., 2018. Fast Approximate Natural Gradient Descent in a Kronecker Factored Eigenbasis, in: Advances in Neural Information Processing Systems. Curran Associates, Inc.\u00a0\u21a9</p> </li> <li> <p>Martens, J., Grosse, R., 2015. Optimizing Neural Networks with Kronecker-factored Approximate Curvature, in: Proceedings of the 32nd International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 2408\u20132417.\u00a0\u21a9</p> </li> <li> <p>Hataya, R., Yamada, M., 2023. Nystr\u00f6m Method for Accurate and Scalable Implicit Differentiation, in: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 4643\u20134654.\u00a0\u21a9</p> </li> </ol>"},{"location":"influence/","title":"The influence function","text":""},{"location":"influence/#the-influence-function","title":"The influence function","text":"<p>Warning</p> <p>The code in the package pydvl.influence is experimental. Package structure and basic API are bound to change before v1.0.0</p> <p>The influence function (IF) is a method to quantify the effect (influence) that each training point has on the parameters of a model, and by extension on any function thereof. In particular, it allows to estimate how much each training sample affects the error on a test point, making the IF useful for understanding and debugging models.</p> <p>Alas, the influence function relies on some assumptions that can make their application difficult. Yet another drawback is that they require the computation of the inverse of the Hessian of the model wrt. its parameters, which is intractable for large models like deep neural networks. Much of the recent research tackles this issue using approximations, like a Neuman series  (Agarwal et al., 2017)<sup>1</sup>, with the most successful solution using a low-rank approximation that iteratively finds increasing eigenspaces of the Hessian  (Schioppa et al., 2022)<sup>2</sup>.</p> <p>pyDVL implements several methods for the efficient computation of the IF for machine learning. In the examples we document some of the difficulties that can arise when using the IF.</p>"},{"location":"influence/#construction","title":"Construction","text":"<p>First introduced in the context of robust statistics in (Hampel, 1974)<sup>3</sup>, the IF was popularized in the context of machine learning in  (Koh and Liang, 2017)<sup>4</sup>.</p> <p>Following their formulation, consider an input space \\(\\mathcal{X}\\) (e.g. images) and an output space \\(\\mathcal{Y}\\) (e.g. labels). Let's take \\(z_i = (x_i, y_i)\\), for \\(i \\in  \\{1,...,n\\}\\) to be the \\(i\\)-th training point, and \\(\\theta\\) to be the (potentially highly) multi-dimensional parameters of a model (e.g. \\(\\theta\\) is a big array with all of a neural network's parameters, including biases and/or dropout rates). We will denote with \\(L(z, \\theta)\\) the loss of the model for point \\(z\\) when the parameters are \\(\\theta.\\)</p> <p>To train a model, we typically minimize the loss over all \\(z_i\\), i.e. the optimal parameters are</p> \\[\\hat{\\theta} = \\arg \\min_\\theta \\sum_{i=1}^n L(z_i, \\theta).\\] <p>In practice, lack of convexity means that one doesn't really obtain the minimizer of the loss, and the training is stopped when the validation loss stops decreasing.</p> <p>For notational convenience, let's define</p> \\[\\hat{\\theta}_{-z} = \\arg \\min_\\theta \\sum_{z_i \\ne z} L(z_i, \\theta), \\] <p>i.e. \\(\\hat{\\theta}_{-z}\\) are the model parameters that minimize the total loss when \\(z\\) is not in the training dataset.</p> <p>In order to compute the impact of each training point on the model, we would need to calculate \\(\\hat{\\theta}_{-z}\\) for each \\(z\\) in the training dataset, thus re-training the model at least ~\\(n\\) times (more if model training is stochastic). This is computationally very expensive, especially for big neural networks. To circumvent this problem, we can just calculate a first order approximation of \\(\\hat{\\theta}\\). This can be done through single backpropagation and without re-training the full model.</p> <p>pyDVL supports two ways of computing the empirical influence function, namely up-weighting of samples and perturbation influences.</p>"},{"location":"influence/#approximating-the-influence-of-a-point","title":"Approximating the influence of a point","text":"<p>Let's define</p> \\[\\hat{\\theta}_{\\epsilon, z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) + \\epsilon L(z, \\theta), \\] <p>which is the optimal \\(\\hat{\\theta}\\) when we up-weight \\(z\\) by an amount \\(\\epsilon \\gt 0\\).</p> <p>From a classical result (a simple derivation is available in Appendix A of  (Koh and Liang, 2017)<sup>4</sup>), we know that:</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta L(z, \\hat{\\theta}), \\] <p>where \\(H_{\\hat{\\theta}} = \\frac{1}{n} \\sum_{i=1}^n \\nabla_\\theta^2 L(z_i, \\hat{\\theta})\\) is the Hessian of \\(L\\). These quantities are also knows as influence factors.</p> <p>Importantly, notice that this expression is only valid when \\(\\hat{\\theta}\\) is a minimum of \\(L\\), or otherwise \\(H_{\\hat{\\theta}}\\) cannot be inverted! At the same time, in machine learning full convergence is rarely achieved, so direct Hessian inversion is not possible. Approximations need to be developed that circumvent the problem of inverting the Hessian of the model in all those (frequent) cases where it is not positive definite.</p> <p>The influence of training point \\(z\\) on test point \\(z_{\\text{test}}\\) is defined as:</p> \\[\\mathcal{I}(z, z_{\\text{test}}) =  L(z_{\\text{test}}, \\hat{\\theta}_{-z}) - L(z_{\\text{test}}, \\hat{\\theta}). \\] <p>Notice that \\(\\mathcal{I}\\) is higher for points \\(z\\) which positively impact the model score, since the loss is higher when they are excluded from training. In practice, one needs to rely on the following infinitesimal approximation:</p> \\[\\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\frac{d L(z_{\\text{test}}, \\hat{\\theta}_{\\epsilon, z})}{d \\epsilon} \\Big|_{\\epsilon=0} \\] <p>Using the chain rule and the results calculated above, we get:</p> \\[\\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_\\theta L(z, \\hat{\\theta}) \\] <p>All the resulting factors are gradients of the loss wrt. the model parameters \\(\\hat{\\theta}\\). This can be easily computed through one or more backpropagation passes.</p>"},{"location":"influence/#perturbation-definition-of-the-influence-score","title":"Perturbation definition of the influence score","text":"<p>How would the loss of the model change if, instead of up-weighting an individual point \\(z\\), we were to up-weight only a single feature of that point? Given \\(z = (x, y)\\), we can define \\(z_{\\delta} = (x+\\delta, y)\\), where \\(\\delta\\) is a vector of zeros except for a 1 in the position of the feature we want to up-weight. In order to approximate the effect of modifying a single feature of a single point on the model score we can define</p> \\[\\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_{i}, \\theta) + \\epsilon L(z_{\\delta}, \\theta) - \\epsilon L(z, \\theta), \\] <p>Similarly to what was done above, we up-weight point \\(z_{\\delta}\\), but then we also remove the up-weighting for all the features that are not modified by \\(\\delta\\). From the calculations in the previous section, it is then easy to see that</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta \\Big( L(z_{\\delta}, \\hat{\\theta}) - L(z, \\hat{\\theta}) \\Big) \\] <p>and if the feature space is continuous and as \\(\\delta \\to 0\\) we can write</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\ \\nabla_x \\nabla_\\theta L(z, \\hat{\\theta}) \\delta + \\mathcal{o}(\\delta) \\] <p>The influence of each feature of \\(z\\) on the loss of the model can therefore be estimated through the following quantity:</p> \\[\\mathcal{I}_{pert}(z, z_{\\text{test}}) = - \\lim_{\\delta \\to 0} \\ \\frac{1}{\\delta} \\frac{d L(z_{\\text{test}}, \\hat{\\theta}_{\\epsilon, \\ z_{\\delta}, \\ -z})}{d \\epsilon} \\Big|_{\\epsilon=0} \\] <p>which, using the chain rule and the results calculated above, is equal to</p> \\[\\mathcal{I}_{pert}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_x \\nabla_\\theta L(z, \\hat{\\theta}) \\] <p>The perturbation definition of the influence score is not straightforward to understand, but it has a simple interpretation: it tells how much the loss of the model changes when a certain feature of point z is up-weighted. A positive perturbation influence score indicates that the feature might have a positive effect on the accuracy of the model.</p> <p>It is worth noting that the perturbation influence score is a very rough estimate of the impact of a point on the models loss and it is subject to large approximation errors. It can nonetheless be used to build training-set attacks, as done in (Koh and Liang, 2017)<sup>4</sup>.</p>"},{"location":"influence/#computation","title":"Computation","text":"<p>The main abstraction of the library for influence calculation is InfluenceFunctionModel.  On implementations of this abstraction, you can call the method <code>influences</code> to compute influences. </p> <p>pyDVL provides implementations to use with pytorch model in pydvl.influence.torch. For detailed information  on available implementations see the documentation in InfluenceFunctionModel.</p> <p>Given a pre-trained pytorch model and a loss, a basic example would look like</p> <p><pre><code>from torch.utils.data import DataLoader\nfrom pydvl.influence.torch import DirectInfluence\n\ntraining_data_loader = DataLoader(...)\ninfl_model = DirectInfluence(model, loss)\ninfl_model = infl_model.fit(training_data_loader)\n\ninfluences = infl_model.influences(x_test, y_test, x, y)\n</code></pre> for batches \\(z_{\\text{test}} = (x_{\\text{test}}, y_{\\text{test}})\\) and \\(z = (x, y)\\) of data. The result is a tensor with one row per test point in  \\(z_{\\text{test}}\\) and one column per point in \\(z\\).  Thus, each entry \\((i, j)\\) represents the influence of training point \\(z[j]\\) on test point \\(z_{\\text{test}}[i]\\).</p> <p>Warning</p> <p>Compared to the mathematical definitions above, we switch the ordering of \\(z\\) and \\(z_{\\text{test}}\\), in order to make the input ordering consistent with the dimensions of the resulting tensor. More concrete if the first dimension of \\(z_{\\text{test}}\\) is \\(N\\) and that of \\(z\\), the resulting tensor is of shape \\(N \\times M\\)</p> <p>A large positive influence indicates that training point \\(j\\) tends to improve the performance of the model on test point \\(i\\), and vice versa, a large negative influence indicates that training point \\(j\\) tends to worsen the performance of the model on test point \\(i\\).</p>"},{"location":"influence/#hessian-regularization","title":"Hessian regularization","text":"<p>Additionally, and as discussed in the introduction, in machine learning training rarely converges to a global minimum of the loss. Despite good apparent convergence, \\(\\hat{\\theta}\\) might be located in a region with flat curvature or close to a saddle point. In particular, the Hessian might have vanishing eigenvalues making its direct inversion impossible. Certain methods, such as the Arnoldi method are robust against these problems, but most are not.</p> <p>To circumvent this problem, many approximate methods can be implemented. The simplest adds a small hessian perturbation term, i.e. \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\), with \\(\\mathbb{I}\\) being the identity matrix. </p> <pre><code>from torch.utils.data import DataLoader\nfrom pydvl.influence.torch import DirectInfluence\n\ntraining_data_loader = DataLoader(...)\ninfl_model = DirectInfluence(model, loss, hessian_regularization=0.01)\ninfl_model = infl_model.fit(training_data_loader)\n</code></pre> <p>This standard trick ensures that the eigenvalues of \\(H_{\\hat{\\theta}}\\) are bounded away from zero and therefore the matrix is invertible. In order for this regularization not to corrupt the outcome too much, the parameter \\(\\lambda\\) should be as small as possible while still allowing a reliable inversion of \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\).</p>"},{"location":"influence/#perturbation-influences","title":"Perturbation influences","text":"<p>The method of empirical influence computation can be selected with the parameter <code>mode</code>:</p> <p><pre><code>from pydvl.influence import InfluenceMode\n\ninfluences = infl_model.influences(x_test, y_test, x, y,\n                                   mode=InfluenceMode.Perturbation)\n</code></pre> The result is a tensor with at least three dimensions. The first two dimensions are the same as in the case of <code>mode=InfluenceMode.Up</code> case, i.e. one row per test point and one column per training point. The remaining dimensions are the same as the number of input features in the data. Therefore, each entry in the tensor represents the influence of each feature of each training point on each test point.</p>"},{"location":"influence/#influence-factors","title":"Influence factors","text":"<p>The influence factors(refer to the previous section for a definition) are typically the most computationally demanding part of influence calculation. They can be obtained via calling the <code>influence_factors</code> method, saved, and later used  for influence calculation on different subsets of the training dataset.</p> <pre><code>influence_factors = infl_model.influence_factors(x_test, y_test)\ninfluences = infl_model.influences_from_factors(influence_factors, x, y)\n</code></pre> <ol> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. JMLR 18, 1\u201340.\u00a0\u21a9</p> </li> <li> <p>Schioppa, A., Zablotskaia, P., Vilar, D., Sokolov, A., 2022. Scaling Up Influence Functions. Proc. AAAI Conf. Artif. Intell. 36, 8179\u20138186. https://doi.org/10.1609/aaai.v36i8.20791 \u21a9</p> </li> <li> <p>Hampel, F.R., 1974. The Influence Curve and Its Role in Robust Estimation. J. Am. Stat. Assoc. 69, 383\u2013393. https://doi.org/10.2307/2285666 \u21a9</p> </li> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions, in: Proceedings of the 34th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 1885\u20131894.\u00a0\u21a9\u21a9\u21a9</p> </li> </ol>"},{"location":"influence/influence_function_model/","title":"Influence Function Model","text":"<p>In almost every practical application it is not possible to construct, even less invert the complete Hessian in memory. pyDVL offers several implementations of  the interface InfluenceFunctionModel ,  which do not compute the full Hessian (in contrast to DirectInfluence ).</p>"},{"location":"influence/influence_function_model/#conjugate-gradient","title":"Conjugate Gradient","text":"<p>This classical procedure for solving linear systems of equations is an iterative method that does not require the explicit inversion of the Hessian. Instead, it only requires the calculation of Hessian-vector products, making it a good choice for large datasets or models with many parameters. It is nevertheless much slower to converge than the direct inversion method and not as accurate.</p> <p>More info on the theory of conjugate gradient can be found on Wikipedia, or in text books such as (Trefethen and Bau, 1997, Lecture 38)<sup>1</sup>.</p> <p>pyDVL also implements a stable block variant of the conjugate  gradient method, defined in (Ji and Li, 2017)<sup>2</sup>, which solves several right hand sides simultaneously.</p> <p>Optionally, the user can provide a pre-conditioner to improve convergence, such  as a Jacobi pre-conditioner , which is a simple diagonal pre-conditioner  based on Hutchinson's diagonal estimator (Bekas et al., 2007)<sup>3</sup>, or a Nystr\u00f6m approximation based pre-conditioner ,  described in (Frangella et al., 2023)<sup>4</sup>. </p> <pre><code>from pydvl.influence.torch import CgInfluence\nfrom pydvl.influence.torch.pre_conditioner import NystroemPreConditioner\n\nif_model = CgInfluence(\n    model,\n    loss,\n    hessian_regularization=0.0,\n    rtol=1e-7,\n    atol=1e-7,\n    maxiter=None,\n    use_block_cg=True,\n    pre_conditioner=NystroemPreConditioner(rank=10)\n)\nif_model.fit(train_loader)\n</code></pre> <p>The additional optional parameters <code>rtol</code>, <code>atol</code>, <code>maxiter</code>, <code>use_block_cg</code> and  <code>pre_conditioner</code> are respectively, the relative tolerance, the absolute tolerance, the maximum number of iterations,  a flag indicating whether to use block variant of cg and an optional pre-conditioner.</p>"},{"location":"influence/influence_function_model/#linear-time-stochastic-second-order-approximation-lissa","title":"Linear time Stochastic Second-Order Approximation (LiSSA)","text":"<p>The LiSSA method is a stochastic approximation of the inverse Hessian vector product. Compared to conjugate gradient it is faster but less accurate and typically suffers from instability.</p> <p>In order to find the solution of the HVP, LiSSA iteratively approximates the inverse of the Hessian matrix with the following update:</p> \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\] <p>where \\(d\\) and \\(s\\) are a dampening and a scaling factor, which are essential for the convergence of the method and they need to be chosen carefully, and I is the identity matrix. More info on the theory of LiSSA can be found in the original paper (Agarwal et al., 2017)<sup>5</sup>.</p> <pre><code>from pydvl.influence.torch import LissaInfluence\nif_model = LissaInfluence(\n   model,\n   loss,\n   hessian_regularization=0.0 \n   maxiter=1000,\n   dampen=0.0,\n   scale=10.0,\n   h0=None,\n   rtol=1e-4,\n)\nif_model.fit(train_loader)\n</code></pre> <p>with the additional optional parameters <code>maxiter</code>, <code>dampen</code>, <code>scale</code>, <code>h0</code>, and <code>rtol</code>, being the maximum number of iterations, the dampening factor, the scaling factor, the initial guess for the solution and the relative tolerance, respectively.</p>"},{"location":"influence/influence_function_model/#arnoldi","title":"Arnoldi","text":"<p>The Arnoldi method is a Krylov subspace method for approximating dominating eigenvalues and eigenvectors. Under a low rank assumption on the Hessian at a minimizer (which is typically observed for deep neural networks), this approximation captures the essential action of the Hessian. More concretely, for \\(Hx=b\\) the solution is approximated by</p> \\[x \\approx V D^{-1} V^T b\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors. See also  (Schioppa et al., 2022)<sup>6</sup>.</p> <pre><code>from pydvl.influence.torch import ArnoldiInfluence\nif_model = ArnoldiInfluence(\n    model,\n    loss,\n    hessian_regularization=0.0,\n    rank_estimate=10,\n    tol=1e-6,\n)\nif_model.fit(train_loader)\n</code></pre>"},{"location":"influence/influence_function_model/#eigenvalue-corrected-k-fac","title":"Eigenvalue Corrected K-FAC","text":"<p>K-FAC, short for Kronecker-Factored Approximate Curvature, is a method that  approximates the Fisher information matrix FIM of a model.  It is possible to show that for classification models with appropriate loss  functions the FIM is equal to the Hessian of the model\u2019s loss over the dataset.  In this restricted but nonetheless important context K-FAC offers an efficient  way to approximate the Hessian and hence the influence scores.  For more info and details refer to the original paper (Martens and Grosse, 2015)<sup>7</sup>.</p> <p>The K-FAC method is implemented in the class EkfacInfluence .  The following code snippet shows how to use the K-FAC method to calculate the  influence function of a model. Note that, in contrast to the other methods for  influence function calculation, K-FAC does not require the loss function as an  input. This is because the current implementation is only applicable to  classification models with a cross entropy loss function. </p> <p><pre><code>from pydvl.influence.torch import EkfacInfluence\nif_model = EkfacInfluence(\n    model,\n    hessian_regularization=0.0,\n)\nif_model.fit(train_loader)\n</code></pre> Upon initialization, the K-FAC method will parse the model and extract which  layers require grad and which do not. Then it will only calculate the influence  scores for the layers that require grad. The current implementation of the  K-FAC method is only available for linear layers, and therefore if the model  contains non-linear layers that require gradient the K-FAC method will raise a  NotImplementedLayerRepresentationException.</p> <p>A further improvement of the K-FAC method is the Eigenvalue Corrected  K-FAC (EKFAC) method (George et al., 2018)<sup>8</sup>, which allows to further re-fit the  eigenvalues of the Hessian, thus providing a more accurate approximation.  On top of the K-FAC method, the EKFAC method is implemented by setting  <code>update_diagonal=True</code> when initialising EkfacInfluence .  The following code snippet shows how to use the EKFAC method to calculate the  influence function of a model. </p> <pre><code>from pydvl.influence.torch import EkfacInfluence\nif_model = EkfacInfluence(\n    model,\n    update_diagonal=True,\n    hessian_regularization=0.0,\n)\nif_model.fit(train_loader)\n</code></pre>"},{"location":"influence/influence_function_model/#nystrom-sketch-and-solve","title":"Nystr\u00f6m Sketch-and-Solve","text":"<p>This approximation is based on a Nystr\u00f6m low-rank approximation of the form</p> \\[\\begin{align*} H_{\\text{nys}} &amp;= (H\\Omega)(\\Omega^TH\\Omega)^{\\dagger}(H\\Omega)^T \\\\\\ &amp;= U \\Lambda U^T, \\end{align*}\\] <p>where \\((\\cdot)^{\\dagger}\\) denotes the Moore-Penrose inverse, in combination with the Sherman\u2013Morrison\u2013Woodbury formula to calculate the action of its inverse:</p> \\[\\begin{equation*}  (H_{\\text{nys}} + \\lambda I)^{-1}x = U(\\Lambda+\\lambda I)U^Tx + \\frac{1}{\\lambda}(I\u2212UU^T)x, \\end{equation*}\\] <p>see also (Hataya and Yamada, 2023)<sup>9</sup> and (Frangella et al., 2023)<sup>4</sup>. The essential  parameter is the rank of the approximation.</p> <pre><code>from pydvl.influence.torch import NystroemSketchInfluence\nif_model = NystroemSketchInfluence(\n    model,\n    loss,\n    rank=10,\n    hessian_regularization=0.0,\n)\nif_model.fit(train_loader)\n</code></pre> <p>These implementations represent the calculation logic on in memory tensors.  To scale up to large collection of data, we map these influence function models  over these collections. For a detailed discussion see the documentation page Scaling Computation.</p> <ol> <li> <p>Trefethen, L.N., Bau, D., Iii, 1997. Numerical Linear Algebra. Society for Industrial and Applied Mathematics. https://doi.org/10.1137/1.9780898719574 \u21a9</p> </li> <li> <p>Ji, H., Li, Y., 2017. A breakdown-free block conjugate gradient method. Bit Numer Math 57, 379\u2013403. https://doi.org/10.1007/s10543-016-0631-z \u21a9</p> </li> <li> <p>Bekas, C., Kokiopoulou, E., Saad, Y., 2007. An estimator for the diagonal of a matrix. Applied Numerical Mathematics, Numerical Algorithms, Parallelism and Applications (2) 57, 1214\u20131229. https://doi.org/10.1016/j.apnum.2007.01.003 \u21a9</p> </li> <li> <p>Frangella, Z., Tropp, J.A., Udell, M., 2023. Randomized Nystr\u00f6m Preconditioning. SIAM J. Matrix Anal. Appl. 44, 718\u2013752. https://doi.org/10.1137/21M1466244 \u21a9\u21a9</p> </li> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. JMLR 18, 1\u201340.\u00a0\u21a9</p> </li> <li> <p>Schioppa, A., Zablotskaia, P., Vilar, D., Sokolov, A., 2022. Scaling Up Influence Functions. Proc. AAAI Conf. Artif. Intell. 36, 8179\u20138186. https://doi.org/10.1609/aaai.v36i8.20791 \u21a9</p> </li> <li> <p>Martens, J., Grosse, R., 2015. Optimizing Neural Networks with Kronecker-factored Approximate Curvature, in: Proceedings of the 32nd International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 2408\u20132417.\u00a0\u21a9</p> </li> <li> <p>George, T., Laurent, C., Bouthillier, X., Ballas, N., Vincent, P., 2018. Fast Approximate Natural Gradient Descent in a Kronecker Factored Eigenbasis, in: Advances in Neural Information Processing Systems. Curran Associates, Inc.\u00a0\u21a9</p> </li> <li> <p>Hataya, R., Yamada, M., 2023. Nystr\u00f6m Method for Accurate and Scalable Implicit Differentiation, in: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 4643\u20134654.\u00a0\u21a9</p> </li> </ol>"},{"location":"influence/scaling_computation/","title":"Scaling Influence Computation","text":"<p>The implementations of InfluenceFunctionModel provide a convenient way to calculate influences for in memory tensors. </p> <p>Nevertheless, there is a need for computing the influences on batches of data. This might happen, if your input data does not fit into memory (e.g. it is very high-dimensional) or for large models the derivative computations exceed your memory or any combinations of these. For this scenario, we want to map our influence function model over collections of batches (or chunks) of data.</p>"},{"location":"influence/scaling_computation/#sequential","title":"Sequential","text":"<p>The simplest way is to use a double for-loop to iterate over the batches sequentially and collect them. pyDVL provides the simple convenience class SequentialInfluenceCalculator to do this. The batch size should be chosen as large as possible, such that the corresponding batches fit into memory.</p> <p><pre><code>from pydvl.influence import SequentialInfluenceCalculator\nfrom pydvl.influence.torch.util import (\n    NestedTorchCatAggregator, \n    TorchNumpyConverter,\n)\nfrom pydvl.influence.torch import CgInfluence\n\nbatch_size = 10\ntrain_dataloader = DataLoader(..., batch_size=batch_size)\ntest_dataloader = DataLoader(..., batch_size=batch_size)\n\ninfl_model = CgInfluence(model, loss, hessian_regularization=0.01)\ninfl_model = infl_model.fit(train_dataloader)\n\ninfl_calc = SequentialInfluenceCalculator(infl_model)\n\n# this does not trigger the computation\nlazy_influences = infl_calc.influences(test_dataloader, train_dataloader)\n\n# trigger computation and pull the result into main memory, \n# result is the full tensor for all combinations of the two loaders\ninfluences = lazy_influences.compute(aggregator=NestedTorchCatAggregator())\n# or\n# trigger computation and write results chunk-wise to disk using zarr \n# in a sequential manner\nlazy_influences.to_zarr(\"local_path/or/url\", TorchNumpyConverter())\n</code></pre> When invoking the <code>compute</code> method, you have the option to specify a custom aggregator  by implementing NestedSequenceAggregator.  This allows for the aggregation of computed chunks.  Such an approach is particularly beneficial for straightforward aggregation tasks,  commonly seen in sequential computation models.  Examples include operations like concatenation, as implemented in  NestedTorchCatAggregator,  or basic min and max operations. </p> <p>For more intricate aggregations, such as an argmax operation,  it's advisable to use the  DaskInfluenceCalculator  (refer to Parallel for more details). This is because it returns data structures in the  form of dask.array.Array objects, which offer an API almost fully  compatible with NumPy arrays.</p>"},{"location":"influence/scaling_computation/#parallel","title":"Parallel","text":"<p>While the sequential calculation helps in the case the resulting tensors are too large to fit into memory,  the batches are computed one after another. Because the influence computation itself is completely data parallel, you may want to use a parallel processing framework. </p> <p>pyDVL provides an implementation of a parallel computation model using dask. The wrapper class DaskInfluenceCalculator has convenience methods to map the influence function computation over chunks of data in a parallel manner.</p> <p>Again, choosing an appropriate chunk size can be crucial. For a better understanding see the official  dask best practice documentation and the following blog entry.</p> <p>Warning</p> <p>Make sure to set <code>threads_per_worker=1</code>, when using the distributed scheduler for computing, if your implementation of InfluenceFunctionModel is not thread-safe. <pre><code>client = Client(threads_per_worker=1)\n</code></pre> For details on dask schedulers see the official documentation.</p> <p><pre><code>import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom pydvl.influence import DaskInfluenceCalculator\nfrom pydvl.influence.torch import CgInfluence\nfrom pydvl.influence.torch.util import (\n    torch_dataset_to_dask_array,\n    TorchNumpyConverter,\n)\nfrom distributed import Client\n\ntrain_data_set: Dataset = LargeDataSet(\n    ...)  # Possible some out of memory large Dataset\ntest_data_set: Dataset = LargeDataSet(\n    ...)  # Possible some out of memory large Dataset\n\ntrain_dataloader = DataLoader(train_data_set)\ninfl_model = CgInfluence(model, loss, hessian_regularization=0.01)\ninfl_model = infl_model.fit(train_dataloader)\n\n# wrap your input data into dask arrays\nchunk_size = 10\nda_x, da_y = torch_dataset_to_dask_array(train_data_set, chunk_size=chunk_size)\nda_x_test, da_y_test = torch_dataset_to_dask_array(test_data_set,\n                                                   chunk_size=chunk_size)\n\n# use only one thread for scheduling, \n# due to non-thread safety of some torch operations\nclient = Client(n_workers=4, threads_per_worker=1)\n\ninfl_calc = DaskInfluenceCalculator(infl_model, \n                                  converter=TorchNumpyConverter(\n                                      device=torch.device(\"cpu\")\n                                  ),\n                                  client=client)\nda_influences = infl_calc.influences(da_x_test, da_y_test, da_x, da_y)\n# da_influences is a dask.array.Array\n# trigger computation and write chunks to disk in parallel\nda_influences.to_zarr(\"path/or/url\")\n</code></pre> During initialization of the  DaskInfluenceCalculator,  the system verifies if all workers are operating in single-threaded mode when the provided influence_function_model is designated as not thread-safe (indicated by the <code>is_thread_safe</code> property). If this condition is not met, the initialization will raise a specific error, signaling a potential thread-safety conflict.</p> <p>To intentionally skip this safety check (e.g., for debugging purposes using the single machine synchronous scheduler), you can supply the DisableClientSingleThreadCheck type.</p> <pre><code>from pydvl.influence import DisableClientSingleThreadCheck\n\ninfl_calc = DaskInfluenceCalculator(infl_model,\n                                    TorchNumpyConverter(device=torch.device(\"cpu\")),\n                                    DisableClientSingleThreadCheck)\nda_influences = infl_calc.influences(da_x_test, da_y_test, da_x, da_y)\nda_influences.compute(scheduler=\"synchronous\")\n</code></pre>"},{"location":"value/","title":"Data valuation","text":"<p>Info</p> <p>If you want to jump right into it, skip ahead to Computing data values. If you want a quick list of applications, see Applications of data valuation. For a list of all algorithms implemented in pyDVL, see Methods.</p> <p>Data valuation is the task of assigning a number to each element of a training set which reflects its contribution to the final performance of some model trained on it. Some methods attempt to be model-agnostic, but in most cases the model is an integral part of the method. In these cases, this number is not an intrinsic property of the element of interest, but typically a function of three factors:</p> <ol> <li> <p>The dataset \\(D\\), or more generally, the distribution it was sampled from: In    some cases one only cares about values wrt. a given data set, in others    value would ideally be the (expected) contribution of a data point to any    random set \\(D\\) sampled from the same distribution. pyDVL implements methods    of the first kind.</p> </li> <li> <p>The algorithm \\(\\mathcal{A}\\) mapping the data \\(D\\) to some estimator \\(f\\) in a    model class \\(\\mathcal{F}\\). E.g. MSE minimization to find the parameters of a    linear model.</p> </li> <li> <p>The performance metric of interest \\(u\\) for the problem. When value depends on    a model, it must be measured in some way which uses it. E.g. the \\(R^2\\) score    or the negative MSE over a test set. This metric will be computed over a    held-out valuation set.</p> </li> </ol> <p>pyDVL collects algorithms for the computation of data values in this sense, mostly those derived from cooperative game theory. The methods can be found in the package [[pydvl.value]], with support from modules pydvl.utils.dataset and pydvl.utils.utility, as detailed below.</p> <p>Warning</p> <p>Be sure to read the section on the difficulties using data values.</p> <p>There are three main families of methods for data valuation: game-theoretic,  influence-based and intrinsic. As of v0.8.1 pyDVL supports the first two. Here, we focus on game-theoretic concepts and refer to the main documentation on the influence funtion for the second.</p>"},{"location":"value/#game-theoretical-methods","title":"Game theoretical methods","text":"<p>The main contenders in game-theoretic approaches are Shapley values (Ghorbani and Zou, 2019)<sup>1</sup>, (Kwon et al., 2021)<sup>2</sup>,  (Schoch et al., 2022)<sup>3</sup>, their generalization to so-called semi-values by (Kwon and Zou, 2022)<sup>4</sup> and [@wang_data_2022], and the Core (Yan and Procaccia, 2021)<sup>5</sup>. All of these are implemented in pyDVL. For a full list see Methods</p> <p>In these methods, data points are considered players in a cooperative game  whose outcome is the performance of the model when trained on subsets  (coalitions) of the data, measured on a held-out valuation set. This  outcome, or utility, must typically be computed for every subset of  the training set, so that an exact computation is \\(\\mathcal{O} (2^n)\\) in the  number of samples \\(n\\), with each iteration requiring a full re-fitting of the  model using a coalition as training set. Consequently, most methods involve  Monte Carlo approximations, and sometimes approximate utilities which are  faster to compute, e.g. proxy models (Wang et al., 2022)<sup>6</sup> or constant-cost approximations like Neural Tangent Kernels (Wu et al., 2022)<sup>7</sup>.</p> <p>The reasoning behind using game theory is that, in order to be useful, an assignment of value, dubbed valuation function, is usually required to fulfil certain requirements of consistency and \"fairness\". For instance, in some applications value should not depend on the order in which data are considered, or it should be equal for samples that contribute equally to any subset of the data (of equal size). When considering aggregated value for (sub-)sets of data there are additional desiderata, like having a value function that does not increase with repeated samples. Game-theoretic methods are all rooted in axioms that by construction ensure different desiderata, but despite their practical usefulness, none of them are either necessary or sufficient for all applications. For instance, SV methods try to equitably distribute all value among all samples, failing to identify repeated ones as unnecessary, with e.g. a zero value.</p>"},{"location":"value/#computing-data-values","title":"Computing data values","text":"<p>Using pyDVL to compute data values is a simple process that can be broken down into three steps:</p> <ol> <li>Creating a Dataset object from your data.</li> <li>Creating a Utility which ties your model to    the dataset and a scoring function.</li> <li>Computing values with a method of your choice, e.g. via    compute_shapley_values.</li> </ol>"},{"location":"value/#creating-a-dataset","title":"Creating a Dataset","text":"<p>The first item in the tuple \\((D, \\mathcal{A}, u)\\) characterising data value is the dataset. The class Dataset is a simple convenience wrapper for the train and test splits that is used throughout pyDVL. The test set will be used to evaluate a scoring function for the model.</p> <p>It can be used as follows:</p> <pre><code>import numpy as np\nfrom pydvl.utils import Dataset\nfrom sklearn.model_selection import train_test_split\nX, y = np.arange(100).reshape((50, 2)), np.arange(50)\nX_train, X_test, y_train, y_test = train_test_split(\n  X, y, test_size=0.5, random_state=16\n)\ndataset = Dataset(X_train, X_test, y_train, y_test)\n</code></pre> <p>It is also possible to construct Datasets from sklearn toy datasets for illustrative purposes using from_sklearn.</p>"},{"location":"value/#grouping-data","title":"Grouping data","text":"<p>Be it because data valuation methods are computationally very expensive, or because we are interested in the groups themselves, it can be often useful or necessary to group samples to valuate them together. GroupedDataset provides an alternative to Dataset with the same interface which allows this.</p> <p>You can see an example in action in the Spotify notebook, but here's a simple example grouping a pre-existing <code>Dataset</code>. First we construct an array mapping each index in the dataset to a group, then use from_dataset:</p> <pre><code>import numpy as np\nfrom pydvl.utils import GroupedDataset\n\n# Randomly assign elements to any one of num_groups:\ndata_groups = np.random.randint(0, num_groups, len(dataset))\ngrouped_dataset = GroupedDataset.from_dataset(dataset, data_groups)\n</code></pre>"},{"location":"value/#creating-a-utility","title":"Creating a Utility","text":"<p>In pyDVL we have slightly overloaded the name \"utility\" and use it to refer to an object that keeps track of all three items in \\((D, \\mathcal{A}, u)\\). This will be an instance of Utility which, as mentioned, is a convenient wrapper for the dataset, model and scoring function used for valuation methods.</p> <p>Here's a minimal example:</p> <pre><code>import sklearn as sk\nfrom pydvl.utils import Dataset, Utility\n\ndataset = Dataset.from_sklearn(sk.datasets.load_iris())\nmodel = sk.svm.SVC()\nutility = Utility(model, dataset)\n</code></pre> <p>The object <code>utility</code> is a callable that data valuation methods will execute with different subsets of training data. Each call will retrain the model on a subset and evaluate it on the test data using a scoring function. By default, Utility will use <code>model.score()</code>, but it is possible to use any scoring function (greater values must be better). In particular, the constructor accepts the same types as argument as sklearn.model_selection.cross_validate: a string, a scorer callable or None for the default.</p> <pre><code>utility = Utility(model, dataset, \"explained_variance\")\n</code></pre> <p><code>Utility</code> will wrap the <code>fit()</code> method of the model to cache its results. This greatly reduces computation times of Monte Carlo methods. Because of how caching is implemented, it is important not to reuse <code>Utility</code> objects for different datasets. You can read more about setting up the cache in the installation guide, and in the documentation of the caching module.</p>"},{"location":"value/#using-custom-scorers","title":"Using custom scorers","text":"<p>The <code>scoring</code> argument of Utility can be used to specify a custom Scorer object. This is a simple wrapper for a callable that takes a model, and test data and returns a score.</p> <p>More importantly, the object provides information about the range of the score, which is used by some methods by estimate the number of samples necessary, and about what default value to use when the model fails to train.</p> <p>Note</p> <p>The most important property of a <code>Scorer</code> is its default value. Because many models will fail to fit on small subsets of the data, it is important to provide a sensible default value for the score.</p> <p>It is possible to skip the construction of the Scorer when constructing the <code>Utility</code> object. The two following calls are equivalent:</p> <pre><code>from pydvl.utils import Utility, Scorer\n\nutility = Utility(\n   model, dataset, \"explained_variance\", score_range=(-np.inf, 1), default_score=0.0\n)\nutility = Utility(\n   model, dataset, Scorer(\"explained_variance\", range=(-np.inf, 1), default=0.0)\n)\n</code></pre>"},{"location":"value/#learning-the-utility","title":"Learning the utility","text":"<p>Because each evaluation of the utility entails a full retrain of the model with a new subset of the training set, it is natural to try to learn this mapping from subsets to scores. This is the idea behind Data Utility Learning (DUL)  (Wang et al., 2022)<sup>6</sup> and in pyDVL it's as simple as wrapping the <code>Utility</code> inside DataUtilityLearning:</p> <pre><code>from pydvl.utils import Utility, DataUtilityLearning, Dataset\nfrom sklearn.linear_model import LinearRegression, LogisticRegression\nfrom sklearn.datasets import load_iris\n\ndataset = Dataset.from_sklearn(load_iris())\nu = Utility(LogisticRegression(), dataset)\ntraining_budget = 3\nwrapped_u = DataUtilityLearning(u, training_budget, LinearRegression())\n\n# First 3 calls will be computed normally\nfor i in range(training_budget):\n   _ = wrapped_u((i,))\n# Subsequent calls will be computed using the fit model for DUL\nwrapped_u((1, 2, 3))\n</code></pre> <p>As you can see, all that is required is a model to learn the utility itself and the fitting and using of the learned model happens behind the scenes.</p> <p>There is a longer example with an investigation of the results achieved by DUL in a dedicated notebook.</p>"},{"location":"value/#leave-one-out-values","title":"Leave-One-Out values","text":"<p>LOO is the simplest approach to valuation. It assigns to each sample its marginal utility as value:</p> \\[v_u(i) = u(D) \u2212 u(D_{-i}).\\] <p>For notational simplicity, we consider the valuation function as defined over the indices of the dataset \\(D\\), and \\(i \\in D\\) is the index of the sample, \\(D_{-i}\\) is the training set without the sample \\(x_i\\), and \\(u\\) is the utility function. See the section on notation for more.</p> <p>For the purposes of data valuation, this is rarely useful beyond serving as a baseline for benchmarking. Although in some benchmarks it can perform astonishingly well on occasion. One particular weakness is that it does not necessarily correlate with an intrinsic value of a sample: since it is a marginal utility, it is affected by diminishing returns. Often, the training set is large enough for a single sample not to have any significant effect on training performance, despite any qualities it may possess. Whether this is indicative of low value or not depends on each one's goals and definitions, but other methods are typically preferable.</p> <pre><code>from pydvl.value.loo import compute_loo\n\nvalues = compute_loo(utility, n_jobs=-1)\n</code></pre> <p>The return value of all valuation functions is an object of type ValuationResult. This can be iterated over, indexed with integers, slices and Iterables, as well as converted to a pandas.DataFrame.</p>"},{"location":"value/#problems-of-data-values","title":"Problems of data values","text":"<p>There are a number of factors that affect how useful values can be for your project. In particular, regression can be especially tricky, but the particular nature of every (non-trivial) ML problem can have an effect:</p> <ul> <li> <p>Variance of the utility: Classical applications of game theoretic value   concepts operate with deterministic utilities, as do many of the bounds in the   literature. But in ML we use an evaluation of the model on a validation set as a   proxy for the true risk. Even if the utility is bounded, its variance will   affect final values, and even more so any Monte Carlo estimates.   Several works have tried to cope with variance. [@wang_data_2022] prove that by   relaxing one of the Shapley axioms and considering the general class of   semi-values, of which Shapley is an instance, one can prove that a choice of   constant weights is the best one can do in a utility-agnostic setting. This   method, dubbed Data Banzhaf, is available in pyDVL as   compute_banzhaf_semivalues.</p> Averaging repeated utility evaluations <p>One workaround in pyDVL is to configure the caching system to allow multiple evaluations of the utility for every index set. A moving average is  computed and returned once the standard error is small, see CachedFuncConfig. Note however that in practice, the likelihood of cache hits is low, so one would have to force recomputation manually somehow.</p> </li> <li> <p>Unbounded utility: Choosing a scorer for a classifier is simple: accuracy   or some F-score provides a bounded number with a clear interpretation. However,   in regression problems most scores, like \\(R^2\\), are not bounded because   regressors can be arbitrarily bad. This leads to great variability in the   utility for low sample sizes, and hence unreliable Monte Carlo approximations   to the values. Nevertheless, in practice it is only the ranking of samples   that matters, and this tends to be accurate (wrt. to the true ranking) despite   inaccurate values.</p> Squashing scores <p>pyDVL offers a dedicated function composition for scorer functions which can be used to squash a score. The following is defined in module score: <pre><code>import numpy as np\nfrom pydvl.utils import compose_score\n\ndef sigmoid(x: float) -&gt; float:\n  return float(1 / (1 + np.exp(-x)))\n\nsquashed_r2 = compose_score(\"r2\", sigmoid, \"squashed r2\")\n\nsquashed_variance = compose_score(\n  \"explained_variance\", sigmoid, \"squashed explained variance\"\n)\n</code></pre> These squashed scores can prove useful in regression problems, but they can also introduce issues in the low-value regime.</p> </li> <li> <p>Data set size: Computing exact Shapley values is NP-hard, and Monte Carlo   approximations can converge slowly. Massive datasets are thus impractical, at   least with game-theoretical methods. A workaround   is to group samples and investigate their value together. You can do this using   GroupedDataset. There is a fully   worked-out example here. Some algorithms   also provide different sampling strategies to reduce the variance, but due to a   no-free-lunch-type theorem, no single strategy can be optimal for all utilities.   Finally, model specific methods like   kNN-Shapley (Jia et al., 2019)<sup>8</sup>, or   altogether different and typically faster approaches like   Data-OOB (Kwon and Zou, 2023)<sup>9</sup> can also be   used. </p> </li> <li> <p>Model size: Since every evaluation of the utility entails retraining the   whole model on a subset of the data, large models require great amounts of   computation. But also, they will effortlessly interpolate small to medium   datasets, leading to great variance in the evaluation of performance on the   dedicated validation set. One mitigation for this problem is cross-validation,   but this would incur massive computational cost. As of v0.8.1 there are no   facilities in pyDVL for cross-validating the utility (note that this would   require cross-validating the whole value computation).</p> </li> </ul>"},{"location":"value/#notation-and-nomenclature","title":"Notation and nomenclature","text":"<p>Todo</p> <p>Organize this section better and use its content consistently throughout the documentation.</p> <p>The following notation is used throughout the documentation:</p> <p>Let \\(D = \\{x_1, \\ldots, x_n\\}\\) be a training set of \\(n\\) samples.</p> <p>The utility function \\(u:\\mathcal{D} \\rightarrow \\mathbb{R}\\) maps subsets of \\(D\\) to real numbers. In pyDVL, we typically call this mapping a score for consistency with sklearn, and reserve the term utility for the triple of dataset \\(D\\), model \\(f\\) and score \\(u\\), since they are used together to compute the value.</p> <p>The value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is denoted as \\(v_u(x_i)\\) or simply \\(v(i)\\).</p> <p>For any \\(S \\subseteq D\\), we denote by \\(S_{-i}\\) the set of samples in \\(D\\) excluding \\(x_i\\), and \\(S_{+i}\\) denotes the set \\(S\\) with \\(x_i\\) added.</p> <p>The marginal utility of adding sample \\(x_i\\) to a subset \\(S\\) is denoted as \\(\\delta(i) := u(S_{+i}) - u(S)\\).</p> <p>The set \\(D_{-i}^{(k)}\\) contains all subsets of \\(D\\) of size \\(k\\) that do not include sample \\(x_i\\).</p> <ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Rivas, M.A., Zou, J., 2021. Efficient Computation and Analysis of Distributional Shapley Values, in: Proceedings of the 24th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 793\u2013801.\u00a0\u21a9</p> </li> <li> <p>Schoch, S., Xu, H., Ji, Y., 2022. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification, in: Proc. Of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). Presented at the Advances in Neural Information Processing Systems (NeurIPS 2022).\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning, in: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022,. Presented at the AISTATS 2022, PMLR.\u00a0\u21a9</p> </li> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9</p> </li> <li> <p>Wang, T., Yang, Y., Jia, R., 2022. Improving Cooperative Game Theory-based Data Valuation via Data Utility Learning. Presented at the International Conference on Learning Representations (ICLR 2022). Workshop on Socially Responsible Machine Learning, arXiv. https://doi.org/10.48550/arXiv.2107.06336 \u21a9\u21a9</p> </li> <li> <p>Wu, Z., Shu, Y., Low, B.K.H., 2022. DAVINZ: Data Valuation using Deep Neural Networks at Initialization, in: Proceedings of the 39th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 24150\u201324176.\u00a0\u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Gurel, N.M., Li, B., Zhang, C., Spanos, C., Song, D., 2019. Efficient task-specific data valuation for nearest neighbor algorithms. Proc. VLDB Endow. 12, 1610\u20131623. https://doi.org/10.14778/3342263.3342637 \u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2023. Data-OOB: Out-of-bag Estimate as a Simple and Efficient Data Value, in: Proceedings of the 40th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 18135\u201318152.\u00a0\u21a9</p> </li> </ol>"},{"location":"value/classwise-shapley/","title":"Class-wise Shapley","text":"<p>Class-wise Shapley (CWS) (Schoch et al., 2022)<sup>1</sup> offers a Shapley framework tailored for classification problems.  Given a sample \\(x_i\\) with label \\(y_i \\in \\mathbb{N}\\), let \\(D_{y_i}\\) be the subset of \\(D\\) with labels \\(y_i\\), and \\(D_{-y_i}\\) be the complement of \\(D_{y_i}\\) in \\(D\\). The key idea is that the sample \\((x_i, y_i)\\) might improve the overall model performance on \\(D\\), while being detrimental for the performance on \\(D_{y_i},\\) e.g. because of a wrong label. To address this issue, the authors introduced</p> \\[ v_u(i) = \\frac{1}{2^{|D_{-y_i}|}} \\sum_{S_{-y_i}} \\left [ \\frac{1}{|D_{y_i}|}\\sum_{S_{y_i}} \\binom{|D_{y_i}|-1}{|S_{y_i}|}^{-1} \\delta(S_{y_i} | S_{-y_i}) \\right ], \\] <p>where \\(S_{y_i} \\subseteq D_{y_i} \\setminus \\{i\\}\\) and \\(S_{-y_i} \\subseteq D_{-y_i}\\) is arbitrary (in particular, not the complement of \\(S_{y_i}\\)). The function \\(\\delta\\) is called set-conditional marginal Shapley value and is defined as</p> \\[ \\delta(S | C) = u( S_{+i} | C ) \u2212 u(S | C), \\] <p>for any set \\(S\\) such that \\(i \\notin S, C\\) and \\(S \\cap C = \\emptyset\\).</p> <p>In practical applications, estimating this quantity is done both with Monte Carlo sampling of the powerset, and the set of index permutations  (Castro et al., 2009)<sup>2</sup>. Typically, this requires fewer samples than the original Shapley value, although the actual speed-up depends on the model and the dataset.</p> <p>Computing classwise Shapley values</p> <p>Like all other game-theoretic valuation methods, CWS requires a Utility object constructed with model and dataset, with the peculiarity of requiring a specific ClasswiseScorer. The entry point is the function compute_classwise_shapley_values:</p> <pre><code>from pydvl.value import *\n\nmodel = ...\ndata = Dataset(...)\nscorer = ClasswiseScorer(...)\nutility = Utility(model, data, scorer)\nvalues = compute_classwise_shapley_values(\n    utility,\n    done=HistoryDeviation(n_steps=500, rtol=5e-2) | MaxUpdates(5000),\n    truncation=RelativeTruncation(utility, rtol=0.01),\n    done_sample_complements=MaxChecks(1),\n    normalize_values=True\n)\n</code></pre>"},{"location":"value/classwise-shapley/#the-class-wise-scorer","title":"The class-wise scorer","text":"<p>In order to use the classwise Shapley value, one needs to define a ClasswiseScorer. This scorer is defined as</p> \\[ u(S) = f(a_S(D_{y_i})) g(a_S(D_{-y_i})), \\] <p>where \\(f\\) and \\(g\\) are monotonically increasing functions, \\(a_S(D_{y_i})\\) is the in-class accuracy, and \\(a_S(D_{-y_i})\\) is the out-of-class accuracy (the names originate from a choice by the authors to use accuracy, but in principle any other score, like \\(F_1\\) can be used). </p> <p>The authors show that \\(f(x)=x\\) and \\(g(x)=e^x\\) have favorable properties and are therefore the defaults, but we leave the option to set different functions \\(f\\) and \\(g\\) for an exploration with different base scores. </p> <p>The default class-wise scorer</p> <p>Constructing the CWS scorer requires choosing a metric and the functions \\(f\\) and \\(g\\):</p> <pre><code>import numpy as np\nfrom pydvl.value.shapley.classwise import ClasswiseScorer\n\n# These are the defaults\nidentity = lambda x: x\nscorer = ClasswiseScorer(\n    \"accuracy\",\n    in_class_discount_fn=identity,\n    out_of_class_discount_fn=np.exp\n)\n</code></pre> Surface of the discounted utility function <p>The level curves for \\(f(x)=x\\) and \\(g(x)=e^x\\) are depicted below. The lines illustrate the contour lines, annotated with their respective gradients. Level curves of the class-wise utility </p>"},{"location":"value/classwise-shapley/#evaluation","title":"Evaluation","text":"<p>We illustrate the method with two experiments: point removal and noise removal, as well as an analysis of the distribution of the values. For this we employ the nine datasets used in (Schoch et al., 2022)<sup>1</sup>, using the same pre-processing. For images, PCA is used to reduce down to 32 the features found by a pre-trained <code>Resnet18</code> model. Standard loc-scale normalization is performed for all models except gradient boosting, since the latter is not sensitive to the scale of the features.</p> Datasets used for evaluation Dataset Data Type Classes Input Dims OpenML ID Diabetes Tabular 2 8 37 Click Tabular 2 11 1216 CPU Tabular 2 21 197 Covertype Tabular 7 54 1596 Phoneme Tabular 2 5 1489 FMNIST Image 2 32 40996 CIFAR10 Image 2 32 40927 MNIST (binary) Image 2 32 554 MNIST (multi) Image 10 32 554 <p>We show mean and coefficient of variation (CV) \\(\\frac{\\sigma}{\\mu}\\) of an \"inner metric\". The former shows the performance of the method, whereas the latter displays its stability: we normalize by the mean to see the relative effect of the standard deviation. Ideally the mean value is maximal and CV minimal. </p> <p>Finally, we note that for all sampling-based valuation methods the same number of evaluations of the marginal utility was used. This is important to make the algorithms comparable, but in practice one should consider using a more sophisticated stopping criterion.</p>"},{"location":"value/classwise-shapley/#dataset-pruning-for-logistic-regression-point-removal","title":"Dataset pruning for logistic regression (point removal)","text":"<p>In (best-)point removal, one first computes values for the training set and then removes in sequence the points with the highest values. After each removal, the remaining points are used to train the model from scratch and performance is measured on a test set. This produces a curve of performance vs. number of points removed which we show below.</p> <p>As a scalar summary of this curve, (Schoch et al., 2022)<sup>1</sup> define Weighted Accuracy Drop (WAD) as:</p> \\[ \\text{WAD} =  \\sum_{j=1}^{n} \\left ( \\frac{1}{j} \\sum_{i=1}^{j}  a_{T_{-\\{1 \\colon i-1 \\}}}(D) - a_{T_{-\\{1 \\colon i \\}}}(D) \\right) = a_T(D) - \\sum_{j=1}^{n} \\frac{a_{T_{-\\{1 \\colon j \\}}}(D)}{j} , \\] <p>where \\(a_T(D)\\) is the accuracy of the model (trained on \\(T\\)) evaluated on \\(D\\) and \\(T_{-\\{1 \\colon j \\}}\\) is the set \\(T\\) without elements from \\(\\{1, \\dots , j \\}\\).</p> <p>We run the point removal experiment for a logistic regression model five times and compute WAD for each run, then report the mean \\(\\mu_\\text{WAD}\\) and standard deviation \\(\\sigma_\\text{WAD}\\).</p> <p> Mean WAD for best-point removal on logistic regression. Values computed using LOO, CWS, Beta Shapley, and TMCS </p> <p>We see that CWS is competitive with all three other methods. In all problems except <code>MNIST (multi)</code> it outperforms TMCS, while in that case TMCS has a slight advantage.</p> <p>In order to understand the variability of WAD we look at its coefficient of variation (lower is better):</p> <p> Coefficient of Variation of WAD for best-point removal on logistic regression. Values computed using LOO, CWS, Beta Shapley, and TMCS </p> <p>CWS is not the best method in terms of CV. For <code>CIFAR10</code>, <code>Click</code>, <code>CPU</code> and <code>MNIST (binary)</code> Beta Shapley has the lowest CV. For <code>Diabetes</code>, <code>MNIST (multi)</code> and <code>Phoneme</code> CWS is the winner and for <code>FMNIST</code> and <code>Covertype</code> TMCS takes the lead. Besides LOO, TMCS has the highest relative standard deviation.</p> <p>The following plot shows accuracy vs number of samples removed. Random values serve as a baseline. The shaded area represents the 95% bootstrap confidence interval of the mean across 5 runs.</p> <p> Accuracy after best-sample removal using values from logistic  regression </p> <p>Because samples are removed from high to low valuation order, we expect a steep decrease in the curve.</p> <p>Overall we conclude that in terms of mean WAD, CWS and TMCS perform best, with CWS's CV on par with Beta Shapley's, making CWS a competitive method.</p>"},{"location":"value/classwise-shapley/#dataset-pruning-for-a-neural-network-by-value-transfer","title":"Dataset pruning for a neural network by value transfer","text":"<p>Transfer of values from one model to another is probably of greater practical relevance: values are computed using a cheap model and used to prune the dataset before training a more expensive one.</p> <p>The following plot shows accuracy vs number of samples removed for transfer from logistic regression to a neural network. The shaded area represents the 95% bootstrap confidence interval of the mean across 5 runs.</p> <p> Accuracy after sample removal using values transferred from logistic regression to an MLP </p> <p>As in the previous experiment samples are removed from high to low valuation order and hence we expect a steep decrease in the curve. CWS is competitive with the other methods, especially in very unbalanced datasets like <code>Click</code>. In other datasets, like <code>Covertype</code>, <code>Diabetes</code> and <code>MNIST (multi)</code> the performance is on par with TMCS.</p>"},{"location":"value/classwise-shapley/#detection-of-mis-labeled-data-points","title":"Detection of mis-labeled data points","text":"<p>The next experiment tries to detect mis-labeled data points in binary classification tasks. 20% of the indices is flipped at random (we don't consider multi-class datasets because there isn't a unique flipping strategy). The following table shows the mean of the area under the curve (AUC) for five runs.</p> <p> Mean AUC for mis-labeled data point detection. Values computed using LOO, CWS, Beta Shapley, and  TMCS </p> <p>In the majority of cases TMCS has a slight advantage over CWS, except for <code>Click</code>, where CWS has a slight edge, most probably due to the unbalanced nature of the dataset. The following plot shows the CV for the AUC of the five runs.</p> <p> Coefficient of variation of AUC for mis-labeled data point detection. Values computed using LOO, CWS, Beta Shapley, and TMCS </p> <p>In terms of CV, CWS has a clear edge over TMCS and Beta Shapley.</p> <p>Finally, we look at the ROC curves training the classifier on the \\(n\\) first samples in increasing order of valuation (i.e. starting with the worst):</p> <p> Mean ROC across 5 runs with 95% bootstrap CI </p> <p>Although at first sight TMCS seems to be the winner, CWS stays competitive after factoring in running time. For a perfectly balanced dataset, CWS needs on average fewer samples than TCMS.</p>"},{"location":"value/classwise-shapley/#value-distribution","title":"Value distribution","text":"<p>For illustration, we compare the distribution of values computed by TMCS and CWS.</p> <p> Histogram and estimated density of the values computed by TMCS and CWS on all nine datasets </p> <p>For <code>Click</code> TMCS has a multi-modal distribution of values. We hypothesize that this is due to the highly unbalanced nature of the dataset, and notice that CWS has a single mode, leading to its greater performance on this dataset.</p>"},{"location":"value/classwise-shapley/#conclusion","title":"Conclusion","text":"<p>CWS is an effective way to handle classification problems, in particular for unbalanced datasets. It reduces the computing requirements by considering in-class and out-of-class points separately.</p> <ol> <li> <p>Schoch, S., Xu, H., Ji, Y., 2022. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification, in: Proc. Of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). Presented at the Advances in Neural Information Processing Systems (NeurIPS 2022).\u00a0\u21a9\u21a9\u21a9</p> </li> <li> <p>Castro, J., G\u00f3mez, D., Tejada, J., 2009. Polynomial calculation of the Shapley value based on sampling. Computers   &amp; Operations Research, Selected papers presented at the Tenth International Symposium on Locational Decisions (ISOLDE X) 36, 1726\u20131730. https://doi.org/10.1016/j.cor.2008.04.004 \u21a9</p> </li> </ol>"},{"location":"value/semi-values/","title":"Semi-values","text":"<p>SV is a particular case of a more general concept called semi-value, which is a generalization to different weighting schemes. A semi-value is any valuation function with the form:</p> \\[ v_\\text{semi}(i) = \\sum_{i=1}^n w(k) \\sum_{S \\subset D_{-i}^{(k)}} [u(S_{+i}) - u(S)], \\] <p>where the coefficients \\(w(k)\\) satisfy the property:</p> \\[\\sum_{k=1}^n w(k) = 1,\\] <p>the set \\(D_{-i}^{(k)}\\) contains all subsets of \\(D\\) of size \\(k\\) that do not include sample \\(x_i\\), \\(S_{+i}\\) is the set \\(S\\) with \\(x_i\\) added, and \\(u\\) is the utility function.</p> <p>Two instances of this are Banzhaf indices (Wang and Jia, 2023)<sup>1</sup>, and Beta Shapley (Kwon and Zou, 2022)<sup>2</sup>, with better numerical and rank stability in certain situations.</p> <p>Note</p> <p>Shapley values are a particular case of semi-values and can therefore also be computed with the methods described here. However, as of version 0.8.1, we recommend using compute_shapley_values instead, in particular because it implements truncation policies for TMCS.</p>"},{"location":"value/semi-values/#beta-shapley","title":"Beta Shapley","text":"<p>For some machine learning applications, where the utility is typically the performance when trained on a set \\(S \\subset D\\), diminishing returns are often observed when computing the marginal utility of adding a new data point.</p> <p>Beta Shapley is a weighting scheme that uses the Beta function to place more weight on subsets deemed to be more informative. The weights are defined as:</p> \\[ w(k) := \\frac{B(k+\\beta, n-k+1+\\alpha)}{B(\\alpha, \\beta)}, \\] <p>where \\(B\\) is the Beta function, and \\(\\alpha\\) and \\(\\beta\\) are parameters that control the weighting of the subsets. Setting both to 1 recovers Shapley values, and setting \\(\\alpha = 1\\), and \\(\\beta = 16\\) is reported in (Kwon and Zou, 2022)<sup>2</sup> to be a good choice for some applications. Beta Shapley values are available in pyDVL through compute_beta_shapley_semivalues:</p> <pre><code>from pydvl.value import *\n\nutility = Utility(model, data)\nvalues = compute_beta_shapley_semivalues(\n    u=utility, done=AbsoluteStandardError(threshold=1e-4), alpha=1, beta=16\n)\n</code></pre> <p>See however the Banzhaf indices section  for an alternative choice of weights which is reported to work better.</p>"},{"location":"value/semi-values/#banzhaf-indices","title":"Banzhaf indices","text":"<p>As noted in the section Problems of Data Values, the Shapley value can be very sensitive to variance in the utility function. For machine learning applications, where the utility is typically the performance when trained on a set \\(S \\subset D\\), this variance is often largest for smaller subsets \\(S\\). It is therefore reasonable to try reducing the relative contribution of these subsets with adequate weights.</p> <p>One such choice of weights is the Banzhaf index, which is defined as the constant:</p> \\[w(k) := 2^{n-1},\\] <p>for all set sizes \\(k\\). The intuition for picking a constant weight is that for any choice of weight function \\(w\\), one can always construct a utility with higher variance where \\(w\\) is greater. Therefore, in a worst-case sense, the best one can do is to pick a constant weight.</p> <p>The authors of (Wang and Jia, 2023)<sup>1</sup> show that Banzhaf indices are more robust to variance in the utility function than Shapley and Beta Shapley values. They are available in pyDVL through compute_banzhaf_semivalues:</p> <pre><code>from pydvl.value import *\n\nutility = Utility(model, data)\nvalues = compute_banzhaf_semivalues(\n    u=utility, done=AbsoluteStandardError(threshold=1e-4), alpha=1, beta=16\n)\n</code></pre>"},{"location":"value/semi-values/#banzhaf-semi-values-with-msr-sampling","title":"Banzhaf semi-values with MSR sampling","text":"<p>Wang et. al. propose a more sample-efficient method for computing Banzhaf  semivalues in their paper Data Banzhaf: A Robust Data Valuation Framework  for Machine Learning (Wang and Jia, 2023)<sup>1</sup>. This method updates all semivalues per evaluation of the utility (i.e. per model trained) based on whether a  specific data point was included in the data subset or not. The expression  for computing the semivalues is</p> \\[\\hat{\\phi}_{MSR}(i) = \\frac{1}{|\\mathbf{S}_{\\ni i}|} \\sum_{S \\in  \\mathbf{S}_{\\ni i}} U(S) - \\frac{1}{|\\mathbf{S}_{\\not{\\ni} i}|}  \\sum_{S \\in \\mathbf{S}_{\\not{\\ni} i}} U(S)\\] <p>where \\(\\mathbf{S}_{\\ni i}\\) are the subsets that contain the index \\(i\\) and  \\(\\mathbf{S}_{\\not{\\ni} i}\\) are the subsets not containing the index \\(i\\).</p> <p>The function implementing this method is compute_msr_banzhaf_semivalues.</p> <p><pre><code>from pydvl.value import compute_msr_banzhaf_semivalues, RankCorrelation, Utility\n\nutility = Utility(model, data)\nvalues = compute_msr_banzhaf_semivalues(\n  u=utility, done=RankCorrelation(rtol=0.001),\n  )\n</code></pre> For further details on how to use this method and a comparison of the sample  efficiency, we suggest to take a look at the example notebook  msr_banzhaf_spotify.</p>"},{"location":"value/semi-values/#general-semi-values","title":"General semi-values","text":"<p>As explained above, both Beta Shapley and Banzhaf indices are special cases of semi-values. In pyDVL we provide a general method for computing these with any combination of the three ingredients that define a semi-value:</p> <ul> <li>A utility function \\(u\\).</li> <li>A sampling method</li> <li>A weighting scheme \\(w\\).</li> </ul> <p>You can construct any combination of these three ingredients with compute_generic_semivalues. The utility function is the same as for Shapley values, and the sampling method can be any of the types defined in the samplers module. For instance, the following snippet is equivalent to the above:</p> <pre><code>from pydvl.value import *\n\ndata = Dataset(...)\nutility = Utility(model, data)\nvalues = compute_generic_semivalues(\n  sampler=PermutationSampler(data.indices),\n  u=utility,\n  coefficient=beta_coefficient(alpha=1, beta=16),\n  done=AbsoluteStandardError(threshold=1e-4),\n)\n</code></pre> <p>Allowing any coefficient can help when experimenting with models which are more sensitive to changes in training set size. However, Data Banzhaf indices are proven to be the most robust to variance in the utility function, in the sense of rank stability, across a range of models and datasets (Wang and Jia, 2023)<sup>1</sup>. </p> <p>Careful with permutation sampling</p> <p>This generic implementation of semi-values allowing for any combination of sampling and weighting schemes is very flexible and, in principle, it recovers the original Shapley value, so that  compute_shapley_values is no longer necessary. However, it loses the optimization in permutation sampling that reuses the utility computation from the last iteration when iterating over a permutation. This doubles the computation requirements (and slightly increases variance) when using permutation sampling, unless the cache is enabled. In addition, as mentioned above, truncation policies are not supported by this generic implementation (as of v0.8.1). For these reasons it is preferable to use compute_shapley_values whenever not computing other semi-values.</p> <ol> <li> <p>Wang, J.T., Jia, R., 2023. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning, in: Proceedings of The 26th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 6388\u20136421.\u00a0\u21a9\u21a9\u21a9\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning, in: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022,. Presented at the AISTATS 2022, PMLR.\u00a0\u21a9\u21a9</p> </li> </ol>"},{"location":"value/shapley/","title":"Shapley value","text":""},{"location":"value/shapley/#shapley-value","title":"Shapley value","text":"<p>The Shapley method is an approach to compute data values originating in cooperative game theory. Shapley values are a common way of assigning payoffs to each participant in a cooperative game (i.e. one in which players can form coalitions) in a way that ensures that certain axioms are fulfilled.</p> <p>pyDVL implements several methods for the computation and approximation of Shapley values. They can all be accessed via the facade function compute_shapley_values. The supported methods are enumerated in ShapleyMode.</p> <p>Empirically, the most useful method is the so-called Truncated Monte Carlo Shapley (Ghorbani and Zou, 2019)<sup>1</sup>, which is a Monte Carlo approximation of the permutation Shapley value.</p>"},{"location":"value/shapley/#combinatorial-shapley","title":"Combinatorial Shapley","text":"<p>The first algorithm is just a verbatim implementation of the definition. As such it returns as exact a value as the utility function allows (see what this means in Problems of Data Values).</p> <p>The value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is computed as a weighted sum of its marginal utility wrt. every possible coalition of training samples within the training set:</p> \\[ v(i) = \\frac{1}{n} \\sum_{S \\subseteq D_{-i}} \\binom{n-1}{ | S | }^{-1} [u(S_{+i}) \u2212 u(S)] ,\\] <p>where \\(D_{-i}\\) denotes the set of samples in \\(D\\) excluding \\(x_i\\), and \\(S_{+i}\\) denotes the set \\(S\\) with \\(x_i\\) added.</p> <pre><code>from pydvl.value import compute_shapley_values\n\nvalues = compute_shapley_values(utility, mode=\"combinatorial_exact\")\ndf = values.to_dataframe(column='value')\n</code></pre> <p>We can convert the return value to a pandas.DataFrame. and name the column with the results as <code>value</code>. Please refer to the documentation in shapley and ValuationResult for more information.</p>"},{"location":"value/shapley/#monte-carlo-combinatorial-shapley","title":"Monte Carlo Combinatorial Shapley","text":"<p>Because the number of subsets \\(S \\subseteq D_{-i}\\) is \\(2^{ | D | - 1 }\\), one typically must resort to approximations. The simplest one is done via Monte Carlo sampling of the powerset \\(\\mathcal{P}(D)\\). In pyDVL this simple technique is called \"Monte Carlo Combinatorial\". The method has very poor converge rate and others are preferred, but if desired, usage follows the same pattern:</p> <pre><code>from pydvl.value import compute_shapley_values, MaxUpdates\n\nvalues = compute_shapley_values(\n   utility, mode=\"combinatorial_montecarlo\", done=MaxUpdates(1000)\n)\ndf = values.to_dataframe(column='cmc')\n</code></pre> <p>The DataFrames returned by most Monte Carlo methods will contain approximate standard errors as an additional column, in this case named <code>cmc_stderr</code>.</p> <p>Note the usage of the object MaxUpdates as the stop condition. This is an instance of a StoppingCriterion. Other examples are MaxTime and AbsoluteStandardError.</p>"},{"location":"value/shapley/#owen-sampling","title":"Owen sampling","text":"<p>Owen Sampling (Okhrati and Lipani, 2021)<sup>2</sup> is a practical algorithm based on the combinatorial definition. It uses a continuous extension of the utility from \\(\\{0,1\\}^n\\), where a 1 in position \\(i\\) means that sample \\(x_i\\) is used to train the model, to \\([0,1]^n\\). The ensuing expression for Shapley value uses integration instead of discrete weights:</p> \\[ v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{-i})} [u(S_{+i}) - u(S)]. \\] <p>Using Owen sampling follows the same pattern as every other method for Shapley values in pyDVL. First construct the dataset and utility, then call compute_shapley_values:</p> <pre><code>from pydvl.value import compute_shapley_values\n\nvalues = compute_shapley_values(\n   u=utility, mode=\"owen\", n_iterations=4, max_q=200\n)\n</code></pre> <p>There are more details on Owen sampling, and its variant Antithetic Owen Sampling in the documentation for the function doing the work behind the scenes: owen_sampling_shapley.</p> <p>Note that in this case we do not pass a StoppingCriterion to the function, but instead the number of iterations and the maximum number of samples to use in the integration.</p>"},{"location":"value/shapley/#permutation-shapley","title":"Permutation Shapley","text":"<p>An equivalent way of computing Shapley values (<code>ApproShapley</code>) appeared in  (Castro et al., 2009)<sup>3</sup> and is the basis for the method most often used in practice. It uses permutations over indices instead of subsets:</p> \\[ v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{:i} \\cup \\{x_i\\}) \u2212 u(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears. To approximate this sum (which has \\(\\mathcal{O}(n!)\\) terms!) one uses Monte Carlo sampling of permutations, something which has surprisingly low sample complexity. One notable difference wrt. the combinatorial approach above is that the approximations always fulfill the efficiency axiom of Shapley, namely \\(\\sum_{i=1}^n \\hat{v}_i = u(D)\\) (see  (Castro et al., 2009)<sup>3</sup>, Proposition 3.2).</p> <p>By adding two types of early stopping, the result is the so-called Truncated Monte Carlo Shapley (Ghorbani and Zou, 2019)<sup>1</sup>, which is efficient enough to be useful in applications. The first is simply a convergence criterion, of which there are several to choose from. The second is a criterion to truncate the iteration over single permutations. RelativeTruncation chooses to stop iterating over samples in a permutation when the marginal utility becomes too small.</p> <pre><code>from pydvl.value import compute_shapley_values, MaxUpdates, RelativeTruncation\n\nvalues = compute_shapley_values(\n    u=utility,\n    mode=\"permutation_montecarlo\",\n    done=MaxUpdates(1000),\n    truncation=RelativeTruncation(utility, rtol=0.01)\n)\n</code></pre> <p>You can see this method in action in this example using the Spotify dataset.</p>"},{"location":"value/shapley/#exact-shapley-for-knn","title":"Exact Shapley for KNN","text":"<p>It is possible to exploit the local structure of K-Nearest Neighbours to reduce the amount of subsets to consider: because no sample besides the K closest affects the score, most are irrelevant and it is possible to compute a value in linear time. This method was introduced by (Jia et al., 2019)<sup>4</sup>, and can be used in pyDVL with:</p> <pre><code>from pydvl.utils import Dataset, Utility\nfrom pydvl.value import compute_shapley_values\nfrom sklearn.neighbors import KNeighborsClassifier\n\nmodel = KNeighborsClassifier(n_neighbors=5)\ndata = Dataset(...)\nutility = Utility(model, data)\nvalues = compute_shapley_values(u=utility, mode=\"knn\")\n</code></pre>"},{"location":"value/shapley/#group-testing","title":"Group testing","text":"<p>An alternative method for the approximation of Shapley values introduced in  (Jia et al., 2019)<sup>4</sup> first estimates the differences of values with a Monte Carlo sum. With</p> \\[\\hat{\\Delta}_{i j} \\approx v_i - v_j,\\] <p>one then solves the following linear constraint satisfaction problem (CSP) to infer the final values:</p> \\[ \\begin{array}{lll} \\sum_{i = 1}^N v_i &amp; = &amp; U (D)\\\\ | v_i - v_j - \\hat{\\Delta}_{i j} | &amp; \\leqslant &amp; \\frac{\\varepsilon}{2 \\sqrt{N}} \\end{array} \\] <p>Warning</p> <p>We have reproduced this method in pyDVL for completeness and benchmarking, but we don't advocate its use because of the speed and memory cost. Despite our best efforts, the number of samples required in practice for convergence can be several orders of magnitude worse than with e.g. TMCS. Additionally, the CSP can sometimes turn out to be infeasible.</p> <p>Usage follows the same pattern as every other Shapley method, but with the addition of an <code>epsilon</code> parameter required for the solution of the CSP. It should be the same value used to compute the minimum number of samples required. This can be done with num_samples_eps_delta, but note that the number returned will be huge! In practice, fewer samples can be enough, but the actual number will strongly depend on the utility, in particular its variance.</p> <pre><code>from pydvl.utils import Dataset, Utility\nfrom pydvl.value import compute_shapley_values\n\nmodel = ...\ndata = Dataset(...)\nutility = Utility(model, data, score_range=(_min, _max))\nmin_iterations = num_samples_eps_delta(epsilon, delta, n, utility.score_range)\nvalues = compute_shapley_values(\n   u=utility, mode=\"group_testing\", n_iterations=min_iterations, eps=eps\n)\n</code></pre> <ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9\u21a9</p> </li> <li> <p>Okhrati, R., Lipani, A., 2021. A Multilinear Sampling Algorithm to Estimate Shapley Values, in: 2020 25th International Conference on Pattern Recognition (ICPR). Presented at the 2020 25th International Conference on Pattern Recognition (ICPR), IEEE, pp. 7992\u20137999. https://doi.org/10.1109/ICPR48806.2021.9412511 \u21a9</p> </li> <li> <p>Castro, J., G\u00f3mez, D., Tejada, J., 2009. Polynomial calculation of the Shapley value based on sampling. Computers   &amp; Operations Research, Selected papers presented at the Tenth International Symposium on Locational Decisions (ISOLDE X) 36, 1726\u20131730. https://doi.org/10.1016/j.cor.2008.04.004 \u21a9\u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Gurel, N.M., Li, B., Zhang, C., Spanos, C., Song, D., 2019. Efficient task-specific data valuation for nearest neighbor algorithms. Proc. VLDB Endow. 12, 1610\u20131623. https://doi.org/10.14778/3342263.3342637 \u21a9\u21a9</p> </li> </ol>"},{"location":"value/the-core/","title":"Core values","text":"<p>Shapley values define a fair way to distribute payoffs amongst all participants (training points) when they form a grand coalition, i.e. when the model is trained on the whole dataset. But they do not consider the question of stability: under which conditions do all participants in a game form the grand coalition? Are the payoffs distributed in such a way that prioritizes its formation?</p> <p>The Core is another solution concept in cooperative game theory that attempts to ensure stability in the sense that it provides the set of feasible payoffs that cannot be improved upon by a sub-coalition. This can be interesting for some applications of data valuation because it yields values consistent with training on the whole dataset, avoiding the spurious selection of subsets.</p> <p>It satisfies the following 2 properties:</p> <ul> <li> <p>Efficiency:   The payoffs are distributed such that it is not possible to make any   participant better off without making another one worse off.   \\(\\sum_{i \\in D} v(i) = u(D).\\)</p> </li> <li> <p>Coalitional rationality:   The sum of payoffs to the agents in any coalition \\(S\\) is at least as large as   the amount that these agents could earn by forming a coalition on their own.   \\(\\sum_{i \\in S} v(i) \\geq u(S), \\forall S \\subset D.\\)</p> </li> </ul> <p>The Core was first introduced into data valuation by (Yan and Procaccia, 2021)<sup>1</sup>, in the following form.</p>"},{"location":"value/the-core/#least-core-values","title":"Least Core values","text":"<p>Unfortunately, for many cooperative games the Core may be empty. By relaxing the coalitional rationality property by a subsidy \\(e \\gt 0\\), we are then able to find approximate payoffs:</p> \\[ \\sum_{i\\in S} v(i) + e \\geq u(S), \\forall S \\subset D, S \\neq \\emptyset \\ ,\\] <p>The Least Core (LC) values \\(\\{v\\}\\) for utility \\(u\\) are computed by solving the following linear program:</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; e &amp; \\\\ \\text{subject to} &amp; \\sum_{i\\in D} v(i) = u(D) &amp; \\\\ &amp; \\sum_{i\\in S} v(i) + e \\geq u(S) &amp;, \\forall S \\subset D, S \\neq \\emptyset  \\\\ \\end{array} \\] <p>Note that solving this program yields a set of solutions \\(\\{v_j:N \\rightarrow \\mathbb{R}\\}\\), whereas the Shapley value is a single function \\(v\\). In order to obtain a single valuation to use, one breaks ties by solving a quadratic program to select the \\(v\\) in the LC with the smallest \\(\\ell_2\\) norm. This is called the egalitarian least core.</p>"},{"location":"value/the-core/#exact-least-core","title":"Exact Least Core","text":"<p>This first algorithm is just a verbatim implementation of the definition, in compute_least_core_values. It computes all constraints for the linear problem by evaluating the utility on every subset of the training data, and returns as exact a value as the utility function allows (see what this means in Problems of Data Values).</p> <pre><code>from pydvl.value import compute_least_core_values\n\nvalues = compute_least_core_values(utility, mode=\"exact\")\n</code></pre>"},{"location":"value/the-core/#monte-carlo-least-core","title":"Monte Carlo Least Core","text":"<p>Because the number of subsets \\(S \\subseteq D \\setminus \\{i\\}\\) is \\(2^{ | D | - 1 }\\), one typically must resort to approximations.</p> <p>The simplest one consists in using a fraction of all subsets for the constraints.  (Yan and Procaccia, 2021)<sup>1</sup> show that a quantity of order \\(\\mathcal{O}((n - \\log \\Delta ) / \\delta^2)\\) is enough to obtain a so-called \\(\\delta\\)-approximate least core with high probability. I.e. the following property holds with probability \\(1-\\Delta\\) over the choice of subsets:</p> \\[ \\mathbb{P}_{S\\sim D}\\left[\\sum_{i\\in S} v(i) + e^{*} \\geq u(S)\\right] \\geq 1 - \\delta, \\] <p>where \\(e^{*}\\) is the optimal least core subsidy. This approximation is also implemented in compute_least_core_values:</p> <pre><code>from pydvl.value import compute_least_core_values\n\nvalues = compute_least_core_values(\n   utility, mode=\"montecarlo\", n_iterations=n_iterations\n)\n</code></pre> <p>Note</p> <p>Although any number is supported, it is best to choose <code>n_iterations</code> to be at least equal to the number of data points.</p> <p>Because computing the Least Core values requires the solution of a linear and a quadratic problem after computing all the utility values, we offer the possibility of splitting the latter from the former. This is useful when running multiple experiments: use mclc_prepare_problem to prepare a list of problems to solve, then solve them in parallel with lc_solve_problems.</p> <pre><code>from pydvl.value.least_core import mclc_prepare_problem, lc_solve_problems\n\nn_experiments = 10\nproblems = [mclc_prepare_problem(utility, n_iterations=n_iterations)\n    for _ in range(n_experiments)]\nvalues = lc_solve_problems(problems)\n</code></pre>"},{"location":"value/the-core/#method-comparison","title":"Method comparison","text":"<p>The TransferLab team reproduced the results of the original paper in a publication for the 2022 MLRC (Benmerzoug and Benito Delgado, 2023)<sup>2</sup>.</p> <p> Best sample removal on binary image classification </p> <p>Roughly speaking, MCLC performs better in identifying high value points, as measured by best-sample removal tasks. In all other aspects, it performs worse or similarly to TMCS at comparable sample budgets. But using an equal number of subsets is more computationally expensive because of the need to solve large linear and quadratic optimization problems.</p> <p> Worst sample removal on binary image classification </p> <p>For these reasons we recommend some variation of SV like TMCS for outlier detection, data cleaning and pruning, and perhaps MCLC for the selection of interesting points to be inspected for the improvement of data collection or model design.</p> <ol> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9\u21a9</p> </li> <li> <p>Benmerzoug, A., Benito Delgado, M. de, 2023. [Re] If you like Shapley, then you\u2019ll love the core. ReScience C 9. https://doi.org/10.5281/zenodo.8173733 \u21a9</p> </li> </ol>"}]}
\ No newline at end of file
diff --git a/devel/sitemap.xml b/devel/sitemap.xml
index b6f547258..005602427 100644
--- a/devel/sitemap.xml
+++ b/devel/sitemap.xml
@@ -2,457 +2,457 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://pydvl.org/stable/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/CHANGELOG/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/CONTRIBUTING/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/influence/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/influence/array/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/influence/base_influence_function_model/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/influence/influence_calculator/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/influence/torch/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/influence/torch/functional/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/influence/torch/influence_function_model/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/influence/torch/pre_conditioner/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/influence/torch/util/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/parallel/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/parallel/backend/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/parallel/config/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/parallel/map_reduce/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/parallel/backends/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/parallel/backends/joblib/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/parallel/backends/ray/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/parallel/futures/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/parallel/futures/ray/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/reporting/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/reporting/plots/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/reporting/scores/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/config/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/dataset/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/functional/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/numeric/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/progress/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/score/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/status/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/types/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/utility/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/caching/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/caching/base/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/caching/config/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/caching/disk/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/caching/memcached/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/utils/caching/memory/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/games/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/result/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/sampler/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/semivalues/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/stopping/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/least_core/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/least_core/common/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/least_core/montecarlo/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/least_core/naive/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/loo/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/loo/loo/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/oob/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/oob/oob/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/shapley/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/shapley/classwise/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/shapley/common/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/shapley/gt/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/shapley/knn/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/shapley/montecarlo/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/shapley/naive/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/shapley/owen/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/shapley/truncated/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/api/pydvl/value/shapley/types/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/data_oob/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/influence_imagenet/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/influence_sentiment_analysis/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/influence_synthetic/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/influence_wine/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/least_core_basic/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/msr_banzhaf_digits/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/shapley_basic_spotify/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/shapley_knn_flowers/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/examples/shapley_utility_learning/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/getting-started/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/getting-started/advanced-usage/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/getting-started/applications/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/getting-started/benchmarking/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/getting-started/first-steps/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/getting-started/glossary/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/getting-started/methods/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/influence/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/influence/influence_function_model/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/influence/scaling_computation/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/value/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/value/classwise-shapley/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/value/semi-values/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/value/shapley/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pydvl.org/stable/value/the-core/</loc>
-         <lastmod>2024-04-12</lastmod>
+         <lastmod>2024-04-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/devel/sitemap.xml.gz b/devel/sitemap.xml.gz
index 085d0deb0..80ebe4c18 100644
Binary files a/devel/sitemap.xml.gz and b/devel/sitemap.xml.gz differ