Update documentation

janclemenslab · Oct 10, 2023 · fc292e0 · fc292e0
1 parent 0bf7def
commit fc292e0
Show file tree

Hide file tree

Showing 73 changed files with 3,035 additions and 935 deletions.
diff --git a/_modules/das/evaluate.html b/_modules/das/evaluate.html
@@ -14,6 +14,7 @@
     <link rel="stylesheet" type="text/css" href="../../_static/panels-bootstrap.5fd3999ee7762ccc51105388f4a9d115.css" />
     <link rel="stylesheet" type="text/css" href="../../_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
     <link rel="stylesheet" type="text/css" href="../../_static/sphinxcontrib-images/LightBox2/lightbox2/dist/css/lightbox.css" />
+    <link rel="stylesheet" type="text/css" href="../../_static/sphinxcontrib-images/LightBox2/lightbox2/dist/css/lightbox.css" />
     <link rel="stylesheet" type="text/css" href="../../_static/styles/furo-extensions.css?digest=30d1aed668e5c3a91c3e3bf6a60b675221979f0e" />
 
 
@@ -392,7 +393,7 @@ <h1>Source code for das.evaluate</h1><div class="highlight"><pre>
 
 
 <div class="viewcode-block" id="evaluate"><a class="viewcode-back" href="../../api/das.evaluate.html#das.evaluate.evaluate">[docs]</a><span class="k">def</span> <span class="nf">evaluate</span><span class="p">(</span>
-    <span class="n">mode_save_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+    <span class="n">model_save_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="n">custom_objects</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Callable</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">full_output</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
     <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
@@ -410,7 +411,7 @@ <h1>Source code for das.evaluate</h1><div class="highlight"><pre>
 <span class="sd">        _type_: _description_</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Loading last best model.&quot;</span><span class="p">)</span>
-    <span class="n">model</span><span class="p">,</span> <span class="n">params</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">load_model_and_params</span><span class="p">(</span><span class="n">mode_save_name</span><span class="p">,</span> <span class="n">custom_objects</span><span class="o">=</span><span class="n">custom_objects</span><span class="p">)</span>
+    <span class="n">model</span><span class="p">,</span> <span class="n">params</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">load_model_and_params</span><span class="p">(</span><span class="n">model_save_name</span><span class="p">,</span> <span class="n">custom_objects</span><span class="o">=</span><span class="n">custom_objects</span><span class="p">)</span>
     <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">summary</span><span class="p">())</span>
 
     <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Loading data from </span><span class="si">{</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;data_dir&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
@@ -456,17 +457,17 @@ <h1>Source code for das.evaluate</h1><div class="highlight"><pre>
             <span class="k">return</span> <span class="n">conf_mat</span><span class="p">,</span> <span class="n">report</span></div>
 
 
-<div class="viewcode-block" id="cli_evaluate"><a class="viewcode-back" href="../../api/das.evaluate.html#das.evaluate.cli_evaluate">[docs]</a><span class="k">def</span> <span class="nf">cli_evaluate</span><span class="p">(</span><span class="n">mode_save_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">):</span>
+<div class="viewcode-block" id="cli_evaluate"><a class="viewcode-back" href="../../api/das.evaluate.html#das.evaluate.cli_evaluate">[docs]</a><span class="k">def</span> <span class="nf">cli_evaluate</span><span class="p">(</span><span class="n">model_save_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Evaluate model.</span>
 
 <span class="sd">    Args:</span>
 <span class="sd">        model_save_name (str): Stem of the path for the model (and parameters).</span>
 <span class="sd">                               File to load will be MODEL_SAVE_NAME + _model.h5.</span>
 <span class="sd">        verbose (int): Display progress bar during prediction. Defaults to 1.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">conf_mat</span><span class="p">,</span> <span class="n">report</span><span class="p">,</span> <span class="n">results_dict</span> <span class="o">=</span> <span class="n">evaluate</span><span class="p">(</span><span class="n">mode_save_name</span><span class="p">,</span> <span class="n">full_output</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span> <span class="n">custom_objects</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+    <span class="n">conf_mat</span><span class="p">,</span> <span class="n">report</span><span class="p">,</span> <span class="n">results_dict</span> <span class="o">=</span> <span class="n">evaluate</span><span class="p">(</span><span class="n">model_save_name</span><span class="p">,</span> <span class="n">full_output</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span> <span class="n">custom_objects</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
 
-    <span class="n">save_filename</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">_results.h5&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">mode_save_name</span><span class="p">)</span>
+    <span class="n">save_filename</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">_results.h5&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">model_save_name</span><span class="p">)</span>
     <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;   Saving to </span><span class="si">{</span><span class="n">save_filename</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
     <span class="n">flammkuchen</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">save_filename</span><span class="p">,</span> <span class="n">results_dict</span><span class="p">)</span></div>
 </pre></div>
@@ -509,5 +510,7 @@ <h1>Source code for das.evaluate</h1><div class="highlight"><pre>
     <script src="../../_static/tabs.js"></script>
     <script src="../../_static/sphinxcontrib-images/LightBox2/lightbox2/dist/js/lightbox-plus-jquery.min.js"></script>
     <script src="../../_static/sphinxcontrib-images/LightBox2/lightbox2-customize/jquery-noconflict.js"></script>
+    <script src="../../_static/sphinxcontrib-images/LightBox2/lightbox2/dist/js/lightbox-plus-jquery.min.js"></script>
+    <script src="../../_static/sphinxcontrib-images/LightBox2/lightbox2-customize/jquery-noconflict.js"></script>
     </body>
 </html>
diff --git a/_modules/das/io.html b/_modules/das/io.html
@@ -14,6 +14,7 @@
     <link rel="stylesheet" type="text/css" href="../../_static/panels-bootstrap.5fd3999ee7762ccc51105388f4a9d115.css" />
     <link rel="stylesheet" type="text/css" href="../../_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
     <link rel="stylesheet" type="text/css" href="../../_static/sphinxcontrib-images/LightBox2/lightbox2/dist/css/lightbox.css" />
+    <link rel="stylesheet" type="text/css" href="../../_static/sphinxcontrib-images/LightBox2/lightbox2/dist/css/lightbox.css" />
     <link rel="stylesheet" type="text/css" href="../../_static/styles/furo-extensions.css?digest=30d1aed668e5c3a91c3e3bf6a60b675221979f0e" />
 
 
@@ -272,6 +273,15 @@ <h1>Source code for das.io</h1><div class="highlight"><pre>
 <span class="sd">&quot;&quot;&quot;</span>
 <span class="kn">import</span> <span class="nn">os.path</span>
 <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">npy_dir</span>
+<span class="kn">import</span> <span class="nn">zarr</span>
+<span class="kn">import</span> <span class="nn">h5py</span>
+<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
+
+
+<div class="viewcode-block" id="MemoryMappedDirectoryStore"><a class="viewcode-back" href="../../api/das.io.html#das.io.MemoryMappedDirectoryStore">[docs]</a><span class="k">class</span> <span class="nc">MemoryMappedDirectoryStore</span><span class="p">(</span><span class="n">zarr</span><span class="o">.</span><span class="n">storage</span><span class="o">.</span><span class="n">DirectoryStore</span><span class="p">):</span>
+    <span class="c1"># faster access to zarr files via memmaping: https://gist.github.com/ivirshup/5c7df5ed10517abf6567a6a9af6c7eaa</span>
+    <span class="k">def</span> <span class="nf">_fromfile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fn</span><span class="p">):</span>
+        <span class="k">return</span> <span class="nb">memoryview</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">memmap</span><span class="p">(</span><span class="n">fn</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;r&quot;</span><span class="p">))</span></div>
 
 
 <span class="k">def</span> <span class="nf">_select</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">x_suffix</span><span class="p">,</span> <span class="n">y_suffix</span><span class="p">):</span>
@@ -318,12 +328,9 @@ <h1>Source code for das.io</h1><div class="highlight"><pre>
 
     <span class="n">location</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">normpath</span><span class="p">(</span><span class="n">location</span><span class="p">)</span>  <span class="c1"># remove trailing path separators</span>
     <span class="k">if</span> <span class="n">location</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;.zarr&quot;</span><span class="p">):</span>
-        <span class="kn">import</span> <span class="nn">zarr</span>
-
-        <span class="n">data</span> <span class="o">=</span> <span class="n">zarr</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">location</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;r&quot;</span><span class="p">)</span>
+        <span class="n">store</span> <span class="o">=</span> <span class="n">zarr</span><span class="o">.</span><span class="n">LRUStoreCache</span><span class="p">(</span><span class="n">MemoryMappedDirectoryStore</span><span class="p">(</span><span class="n">location</span><span class="p">),</span> <span class="n">max_size</span><span class="o">=</span><span class="mf">8e9</span><span class="p">)</span>
+        <span class="n">data</span> <span class="o">=</span> <span class="n">zarr</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="n">store</span><span class="o">=</span><span class="n">store</span><span class="p">,</span> <span class="n">overwrite</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
     <span class="k">elif</span> <span class="n">location</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;.h5&quot;</span><span class="p">):</span>
-        <span class="kn">import</span> <span class="nn">h5py</span>
-
         <span class="n">data</span> <span class="o">=</span> <span class="n">h5py</span><span class="o">.</span><span class="n">File</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">location</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;r&quot;</span><span class="p">)</span>
     <span class="k">elif</span> <span class="n">location</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;.npy&quot;</span><span class="p">):</span>
         <span class="n">data</span> <span class="o">=</span> <span class="n">npy_dir</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">location</span><span class="p">)</span>
@@ -377,5 +384,7 @@ <h1>Source code for das.io</h1><div class="highlight"><pre>
     <script src="../../_static/tabs.js"></script>
     <script src="../../_static/sphinxcontrib-images/LightBox2/lightbox2/dist/js/lightbox-plus-jquery.min.js"></script>
     <script src="../../_static/sphinxcontrib-images/LightBox2/lightbox2-customize/jquery-noconflict.js"></script>
+    <script src="../../_static/sphinxcontrib-images/LightBox2/lightbox2/dist/js/lightbox-plus-jquery.min.js"></script>
+    <script src="../../_static/sphinxcontrib-images/LightBox2/lightbox2-customize/jquery-noconflict.js"></script>
     </body>
 </html>