Add changes for 5431f20

mila-iqia · Jul 10, 2024 · b4bf2d8 · b4bf2d8
1 parent e935c80
commit b4bf2d8
Show file tree

Hide file tree

Showing 7 changed files with 343 additions and 1 deletion.
diff --git a/_sources/process.rst.txt b/_sources/process.rst.txt
@@ -0,0 +1,104 @@
+Request For proposal
+====================
+
+Preparing
+---------
+
+1. Make sure milabench support the targetted hardware
+
+   * NVIDIA
+   * AMD
+
+2. Create a milabench configuration for your RFP
+   Milabench comes with a wide variety of benchmarks.
+   You should select and weight each benchmarks according to your
+   target hardware.
+
+.. code-block:: yaml
+
+   include:
+     - base.yaml
+
+   llama:
+     enabled: true
+     weight: 1.0
+
+   resnet50:
+     enabled: true
+     weight: 1.0
+
+
+.. code-block:: yaml
+
+   milabench resolve myconfig.yaml > RFP.yaml
+
+
+3. Prepare a container for your RFP
+
+
+.. code-block::
+
+   FROM milabench:cuda-v1.2.3
+
+   COPY RFP.yaml .../RFP.yaml
+
+   ENV MILABENCH_CONFIG=".../RFP.yaml
+
+   CMD milabench run
+
+
+4.  Hot fixes
+
+   * Disable a benchmarks
+   * update container
+
+
+Vendor Instructions
+-------------------
+
+1. Vendor needs to create a system configuration that will
+   specify the different compute nodes that will be used by milabench
+
+.. code-block::
+
+   system:
+      sshkey: <privatekey>
+      arch: cuda
+      docker_image: ghcr.io/mila-iqia/milabench:cuda-nightly
+
+      nodes:
+         - name: node1
+            ip: 192.168.0.25
+            main: true
+            port: 8123
+            user: <username>
+
+         - name: node2
+            ip: 192.168.0.26
+            main: false
+            user: <username>
+
+
+2. Run milabench
+
+.. code-block::
+
+   export MILABENCH_IMAGE=ghcr.io/mila-iqia/milabench:cuda-nightly
+
+   # create ...
+   mkdir -p configs
+   mkdir -p results
+
+   # put your vendor specific configuration
+   vi configs/system.yaml
+
+   #
+   docker pull $MILABENCH_IMAGE
+
+   # run milabench
+   docker run -it --rm --gpus all --network host --ipc=host --privileged \
+      -v $SSH_KEY_FILE:/milabench/id_milabench        \
+      -v $(pwd)/results:/milabench/envs/runs          \
+      -v $(pwd)/configs:/milabench/envs/configs       \
+      $MILABENCH_IMAGE                                \
+      milabench run --system /milabench/envs/configs/system.yaml
diff --git a/genindex.html b/genindex.html
@@ -186,6 +186,12 @@ <h2 id="P">P</h2>
 <h2 id="R">R</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="ref-pack.html#milabench.pack.Package.resolve_argument">resolve_argument() (milabench.pack.Package method)</a>
+</li>
+  </ul></td>
+  <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="ref-pack.html#milabench.pack.Package.resolve_placeholder">resolve_placeholder() (milabench.pack.Package method)</a>
+</li>
       <li><a href="ref-pack.html#milabench.pack.Package.run">run() (milabench.pack.Package method)</a>
 </li>
   </ul></td>

diff --git a/objects.inv b/objects.inv
diff --git a/process.html b/process.html
@@ -0,0 +1,211 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" data-content_root="./">
+<head>
+  <meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>Request For proposal &mdash; milabench  documentation</title>
+      <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
+      <link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=19f00094" />
+
+
+  <!--[if lt IE 9]>
+    <script src="_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+
+        <script src="_static/jquery.js?v=5d32c60e"></script>
+        <script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+        <script src="_static/documentation_options.js?v=5929fcd5"></script>
+        <script src="_static/doctools.js?v=9a2dae69"></script>
+        <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
+    <script src="_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="genindex.html" />
+    <link rel="search" title="Search" href="search.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+
+
+
+          <a href="index.html" class="icon icon-home">
+            milabench
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="usage.html">Install and use</a></li>
+<li class="toctree-l1"><a class="reference internal" href="docker.html">Docker</a></li>
+<li class="toctree-l1"><a class="reference internal" href="dev-usage.html">Using milabench (DEVELOPERS)</a></li>
+<li class="toctree-l1"><a class="reference internal" href="new_benchmarks.html">Creating a new benchmark</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reference.html">Reference</a></li>
+<li class="toctree-l1"><a class="reference internal" href="sizer.html">Scaling</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="index.html">milabench</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="index.html" class="icon icon-home" aria-label="Home"></a></li>
+      <li class="breadcrumb-item active">Request For proposal</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="_sources/process.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+
+  <section id="request-for-proposal">
+<h1>Request For proposal<a class="headerlink" href="#request-for-proposal" title="Link to this heading"></a></h1>
+<section id="preparing">
+<h2>Preparing<a class="headerlink" href="#preparing" title="Link to this heading"></a></h2>
+<ol class="arabic simple">
+<li><p>Make sure milabench support the targetted hardware</p>
+<ul class="simple">
+<li><p>NVIDIA</p></li>
+<li><p>AMD</p></li>
+</ul>
+</li>
+<li><p>Create a milabench configuration for your RFP
+Milabench comes with a wide variety of benchmarks.
+You should select and weight each benchmarks according to your
+target hardware.</p></li>
+</ol>
+<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">include</span><span class="p">:</span>
+<span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">base.yaml</span>
+
+<span class="nt">llama</span><span class="p">:</span>
+<span class="w">  </span><span class="nt">enabled</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
+<span class="w">  </span><span class="nt">weight</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1.0</span>
+
+<span class="nt">resnet50</span><span class="p">:</span>
+<span class="w">  </span><span class="nt">enabled</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
+<span class="w">  </span><span class="nt">weight</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1.0</span>
+</pre></div>
+</div>
+<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">milabench resolve myconfig.yaml &gt; RFP.yaml</span>
+</pre></div>
+</div>
+<ol class="arabic simple" start="3">
+<li><p>Prepare a container for your RFP</p></li>
+</ol>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">FROM</span> <span class="n">milabench</span><span class="p">:</span><span class="n">cuda</span><span class="o">-</span><span class="n">v1</span><span class="mf">.2.3</span>
+
+<span class="n">COPY</span> <span class="n">RFP</span><span class="o">.</span><span class="n">yaml</span> <span class="o">.../</span><span class="n">RFP</span><span class="o">.</span><span class="n">yaml</span>
+
+<span class="n">ENV</span> <span class="n">MILABENCH_CONFIG</span><span class="o">=</span><span class="s2">&quot;.../RFP.yaml</span>
+
+<span class="n">CMD</span> <span class="n">milabench</span> <span class="n">run</span>
+</pre></div>
+</div>
+<ol class="arabic simple" start="4">
+<li><p>Hot fixes</p></li>
+</ol>
+<blockquote>
+<div><ul class="simple">
+<li><p>Disable a benchmarks</p></li>
+<li><p>update container</p></li>
+</ul>
+</div></blockquote>
+</section>
+<section id="vendor-instructions">
+<h2>Vendor Instructions<a class="headerlink" href="#vendor-instructions" title="Link to this heading"></a></h2>
+<ol class="arabic simple">
+<li><p>Vendor needs to create a system configuration that will
+specify the different compute nodes that will be used by milabench</p></li>
+</ol>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">system</span><span class="p">:</span>
+   <span class="n">sshkey</span><span class="p">:</span> <span class="o">&lt;</span><span class="n">privatekey</span><span class="o">&gt;</span>
+   <span class="n">arch</span><span class="p">:</span> <span class="n">cuda</span>
+   <span class="n">docker_image</span><span class="p">:</span> <span class="n">ghcr</span><span class="o">.</span><span class="n">io</span><span class="o">/</span><span class="n">mila</span><span class="o">-</span><span class="n">iqia</span><span class="o">/</span><span class="n">milabench</span><span class="p">:</span><span class="n">cuda</span><span class="o">-</span><span class="n">nightly</span>
+
+   <span class="n">nodes</span><span class="p">:</span>
+      <span class="o">-</span> <span class="n">name</span><span class="p">:</span> <span class="n">node1</span>
+         <span class="n">ip</span><span class="p">:</span> <span class="mf">192.168.0.25</span>
+         <span class="n">main</span><span class="p">:</span> <span class="n">true</span>
+         <span class="n">port</span><span class="p">:</span> <span class="mi">8123</span>
+         <span class="n">user</span><span class="p">:</span> <span class="o">&lt;</span><span class="n">username</span><span class="o">&gt;</span>
+
+      <span class="o">-</span> <span class="n">name</span><span class="p">:</span> <span class="n">node2</span>
+         <span class="n">ip</span><span class="p">:</span> <span class="mf">192.168.0.26</span>
+         <span class="n">main</span><span class="p">:</span> <span class="n">false</span>
+         <span class="n">user</span><span class="p">:</span> <span class="o">&lt;</span><span class="n">username</span><span class="o">&gt;</span>
+</pre></div>
+</div>
+<ol class="arabic simple" start="2">
+<li><p>Run milabench</p></li>
+</ol>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>export MILABENCH_IMAGE=ghcr.io/mila-iqia/milabench:cuda-nightly
+
+# create ...
+mkdir -p configs
+mkdir -p results
+
+# put your vendor specific configuration
+vi configs/system.yaml
+
+#
+docker pull $MILABENCH_IMAGE
+
+# run milabench
+docker run -it --rm --gpus all --network host --ipc=host --privileged \
+   -v $SSH_KEY_FILE:/milabench/id_milabench        \
+   -v $(pwd)/results:/milabench/envs/runs          \
+   -v $(pwd)/configs:/milabench/envs/configs       \
+   $MILABENCH_IMAGE                                \
+   milabench run --system /milabench/envs/configs/system.yaml
+</pre></div>
+</div>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2022, Mila IDT.</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
diff --git a/ref-pack.html b/ref-pack.html
@@ -70,6 +70,8 @@
 <li class="toctree-l4"><a class="reference internal" href="#milabench.pack.Package.make_env"><code class="docutils literal notranslate"><span class="pre">Package.make_env()</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="#milabench.pack.Package.pin"><code class="docutils literal notranslate"><span class="pre">Package.pin()</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="#milabench.pack.Package.prepare"><code class="docutils literal notranslate"><span class="pre">Package.prepare()</span></code></a></li>
+<li class="toctree-l4"><a class="reference internal" href="#milabench.pack.Package.resolve_argument"><code class="docutils literal notranslate"><span class="pre">Package.resolve_argument()</span></code></a></li>
+<li class="toctree-l4"><a class="reference internal" href="#milabench.pack.Package.resolve_placeholder"><code class="docutils literal notranslate"><span class="pre">Package.resolve_placeholder()</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="#milabench.pack.Package.run"><code class="docutils literal notranslate"><span class="pre">Package.run()</span></code></a></li>
 </ul>
 </li>
@@ -303,6 +305,23 @@
 <p>The default value of <code class="docutils literal notranslate"><span class="pre">self.prepare_script</span></code> is <code class="docutils literal notranslate"><span class="pre">&quot;prepare.py&quot;</span></code>.</p>
 </dd></dl>
 
+<dl class="py method">
+<dt class="sig sig-object py" id="milabench.pack.Package.resolve_argument">
+<span class="sig-name descname"><span class="pre">resolve_argument</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#milabench.pack.Package.resolve_argument" title="Link to this definition"></a></dt>
+<dd><p>Resolve as single placeholder argument</p>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="milabench.pack.Package.resolve_placeholder">
+<span class="sig-name descname"><span class="pre">resolve_placeholder</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">placeholder</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#milabench.pack.Package.resolve_placeholder" title="Link to this definition"></a></dt>
+<dd><p>Resolve as single placeholder argument</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">resolve_placeholder</span><span class="p">(</span><span class="s2">&quot;auto(</span><span class="si">{n_worker}</span><span class="s2">, 8)&quot;</span><span class="p">)</span>
+<span class="go">16</span>
+</pre></div>
+</div>
+</dd></dl>
+
 <dl class="py method">
 <dt class="sig sig-object py" id="milabench.pack.Package.run">
 <em class="property"><span class="pre">async</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">run</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#milabench.pack.Package.run" title="Link to this definition"></a></dt>

diff --git a/reference.html b/reference.html
@@ -104,6 +104,8 @@ <h1>Reference<a class="headerlink" href="#reference" title="Link to this heading
 <li class="toctree-l3"><a class="reference internal" href="ref-pack.html#milabench.pack.Package.make_env"><code class="docutils literal notranslate"><span class="pre">Package.make_env()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="ref-pack.html#milabench.pack.Package.pin"><code class="docutils literal notranslate"><span class="pre">Package.pin()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="ref-pack.html#milabench.pack.Package.prepare"><code class="docutils literal notranslate"><span class="pre">Package.prepare()</span></code></a></li>
+<li class="toctree-l3"><a class="reference internal" href="ref-pack.html#milabench.pack.Package.resolve_argument"><code class="docutils literal notranslate"><span class="pre">Package.resolve_argument()</span></code></a></li>
+<li class="toctree-l3"><a class="reference internal" href="ref-pack.html#milabench.pack.Package.resolve_placeholder"><code class="docutils literal notranslate"><span class="pre">Package.resolve_placeholder()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="ref-pack.html#milabench.pack.Package.run"><code class="docutils literal notranslate"><span class="pre">Package.run()</span></code></a></li>
 </ul>
 </li>

diff --git a/searchindex.js b/searchindex.js