Skip to content

Commit

Permalink
deploy: c05c9e9
Browse files Browse the repository at this point in the history
  • Loading branch information
@Az-r-ow
Az-r-ow committed Jul 8, 2024
1 parent f51b1a8 commit 1b85deb
Showing 1 changed file with 53 additions and 50 deletions.
103 changes: 53 additions & 50 deletions cpp-docs/Adam_8hpp_source.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,60 +145,63 @@
<div class="line"><a id="l00058" name="l00058"></a><span class="lineno"> 58</span> v = Eigen::MatrixBase&lt;Derived1&gt;::Zero(param.rows(), param.cols());</div>
<div class="line"><a id="l00059" name="l00059"></a><span class="lineno"> 59</span> }</div>
<div class="line"><a id="l00060" name="l00060"></a><span class="lineno"> 60</span> </div>
<div class="line"><a id="l00061" name="l00061"></a><span class="lineno"> 61</span> <span class="comment">// update biased first moment estimate</span></div>
<div class="line"><a id="l00062" name="l00062"></a><span class="lineno"> 62</span> m = (beta1 * m).array() + ((1 - beta2) * gradients.array()).array();</div>
<div class="line"><a id="l00061" name="l00061"></a><span class="lineno"> 61</span> assert(gradients.rows() == m.rows() &amp;&amp; gradients.cols() == m.cols());</div>
<div class="line"><a id="l00062" name="l00062"></a><span class="lineno"> 62</span> assert(gradients.rows() == v.rows() &amp;&amp; gradients.cols() == v.cols());</div>
<div class="line"><a id="l00063" name="l00063"></a><span class="lineno"> 63</span> </div>
<div class="line"><a id="l00064" name="l00064"></a><span class="lineno"> 64</span> <span class="comment">// updated biased second raw moment estimate</span></div>
<div class="line"><a id="l00065" name="l00065"></a><span class="lineno"> 65</span> v = (beta2 * v).array() +</div>
<div class="line"><a id="l00066" name="l00066"></a><span class="lineno"> 66</span> ((1 - beta2) * (gradients.array() * gradients.array())).array();</div>
<div class="line"><a id="l00067" name="l00067"></a><span class="lineno"> 67</span> </div>
<div class="line"><a id="l00068" name="l00068"></a><span class="lineno"> 68</span> <span class="comment">// compute bias-corrected first moment estimate</span></div>
<div class="line"><a id="l00069" name="l00069"></a><span class="lineno"> 69</span> <span class="keywordtype">double</span> beta1_t = std::pow(beta1, t);</div>
<div class="line"><a id="l00064" name="l00064"></a><span class="lineno"> 64</span> <span class="comment">// update biased first moment estimate</span></div>
<div class="line"><a id="l00065" name="l00065"></a><span class="lineno"> 65</span> m = (beta1 * m).array() + ((1 - beta2) * gradients.array()).array();</div>
<div class="line"><a id="l00066" name="l00066"></a><span class="lineno"> 66</span> </div>
<div class="line"><a id="l00067" name="l00067"></a><span class="lineno"> 67</span> <span class="comment">// updated biased second raw moment estimate</span></div>
<div class="line"><a id="l00068" name="l00068"></a><span class="lineno"> 68</span> v = (beta2 * v).array() +</div>
<div class="line"><a id="l00069" name="l00069"></a><span class="lineno"> 69</span> ((1 - beta2) * (gradients.array() * gradients.array())).array();</div>
<div class="line"><a id="l00070" name="l00070"></a><span class="lineno"> 70</span> </div>
<div class="line"><a id="l00071" name="l00071"></a><span class="lineno"> 71</span> <span class="comment">// compute bias-corrected second raw moment estimate</span></div>
<div class="line"><a id="l00072" name="l00072"></a><span class="lineno"> 72</span> <span class="keywordtype">double</span> beta2_t = std::pow(beta2, t);</div>
<div class="line"><a id="l00071" name="l00071"></a><span class="lineno"> 71</span> <span class="comment">// compute bias-corrected first moment estimate</span></div>
<div class="line"><a id="l00072" name="l00072"></a><span class="lineno"> 72</span> <span class="keywordtype">double</span> beta1_t = std::pow(beta1, t);</div>
<div class="line"><a id="l00073" name="l00073"></a><span class="lineno"> 73</span> </div>
<div class="line"><a id="l00074" name="l00074"></a><span class="lineno"> 74</span> <span class="keywordtype">double</span> alpha_t = alpha * (sqrt(1 - beta2_t) / (1 - beta1_t));</div>
<div class="line"><a id="l00075" name="l00075"></a><span class="lineno"> 75</span> </div>
<div class="line"><a id="l00076" name="l00076"></a><span class="lineno"> 76</span> <span class="comment">// update param</span></div>
<div class="line"><a id="l00077" name="l00077"></a><span class="lineno"> 77</span> param =</div>
<div class="line"><a id="l00078" name="l00078"></a><span class="lineno"> 78</span> param.array() - alpha_t * (m.array() / (v.array().sqrt() + epsilon));</div>
<div class="line"><a id="l00079" name="l00079"></a><span class="lineno"> 79</span> }</div>
<div class="line"><a id="l00080" name="l00080"></a><span class="lineno"> 80</span> </div>
<div class="line"><a id="l00081" name="l00081"></a><span class="lineno"> 81</span> <span class="keyword">private</span>:</div>
<div class="line"><a id="l00082" name="l00082"></a><span class="lineno"> 82</span> <span class="keywordtype">double</span> beta1;</div>
<div class="line"><a id="l00083" name="l00083"></a><span class="lineno"> 83</span> <span class="keywordtype">double</span> beta2;</div>
<div class="line"><a id="l00084" name="l00084"></a><span class="lineno"> 84</span> <span class="keywordtype">double</span> epsilon;</div>
<div class="line"><a id="l00085" name="l00085"></a><span class="lineno"> 85</span> <span class="keywordtype">int</span> t = 0;</div>
<div class="line"><a id="l00086" name="l00086"></a><span class="lineno"> 86</span> <span class="keywordtype">int</span> cl; <span class="comment">// Current layer (should be initialized to the total number of layers</span></div>
<div class="line"><a id="l00087" name="l00087"></a><span class="lineno"> 87</span> <span class="comment">// - 0)</span></div>
<div class="line"><a id="l00088" name="l00088"></a><span class="lineno"> 88</span> <span class="keywordtype">int</span> ll; <span class="comment">// Last layer (should also be initialized to numLayers - 1)</span></div>
<div class="line"><a id="l00089" name="l00089"></a><span class="lineno"> 89</span> std::vector&lt;Eigen::MatrixXd&gt; mWeights; <span class="comment">// First-moment vector for weights</span></div>
<div class="line"><a id="l00090" name="l00090"></a><span class="lineno"> 90</span> std::vector&lt;Eigen::MatrixXd&gt; vWeights; <span class="comment">// Second-moment vector for weights</span></div>
<div class="line"><a id="l00091" name="l00091"></a><span class="lineno"> 91</span> std::vector&lt;Eigen::MatrixXd&gt; mBiases; <span class="comment">// First-moment vector for biases</span></div>
<div class="line"><a id="l00092" name="l00092"></a><span class="lineno"> 92</span> std::vector&lt;Eigen::MatrixXd&gt; vBiases; <span class="comment">// Second-moment vector for biases</span></div>
<div class="line"><a id="l00093" name="l00093"></a><span class="lineno"> 93</span> </div>
<div class="line"><a id="l00094" name="l00094"></a><span class="lineno"> 94</span> <span class="keywordtype">void</span> insiderInit(<span class="keywordtype">size_t</span> numLayers)<span class="keyword"> override </span>{</div>
<div class="line"><a id="l00095" name="l00095"></a><span class="lineno"> 95</span> cl = numLayers - 1;</div>
<div class="line"><a id="l00096" name="l00096"></a><span class="lineno"> 96</span> ll = numLayers - 1;</div>
<div class="line"><a id="l00097" name="l00097"></a><span class="lineno"> 97</span> </div>
<div class="line"><a id="l00098" name="l00098"></a><span class="lineno"> 98</span> Eigen::MatrixXd dotMatrix = Eigen::MatrixXd::Zero(0, 0);</div>
<div class="line"><a id="l00099" name="l00099"></a><span class="lineno"> 99</span> </div>
<div class="line"><a id="l00100" name="l00100"></a><span class="lineno"> 100</span> <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = mWeights.size(); i &lt; numLayers; i++) {</div>
<div class="line"><a id="l00101" name="l00101"></a><span class="lineno"> 101</span> mWeights.push_back(dotMatrix);</div>
<div class="line"><a id="l00102" name="l00102"></a><span class="lineno"> 102</span> vWeights.push_back(dotMatrix);</div>
<div class="line"><a id="l00103" name="l00103"></a><span class="lineno"> 103</span> mBiases.push_back(dotMatrix);</div>
<div class="line"><a id="l00104" name="l00104"></a><span class="lineno"> 104</span> vBiases.push_back(dotMatrix);</div>
<div class="line"><a id="l00105" name="l00105"></a><span class="lineno"> 105</span> };</div>
<div class="line"><a id="l00106" name="l00106"></a><span class="lineno"> 106</span> }</div>
<div class="line"><a id="l00107" name="l00107"></a><span class="lineno"> 107</span> </div>
<div class="line"><a id="l00108" name="l00108"></a><span class="lineno"> 108</span> <span class="keywordtype">void</span> setCurrentL() {</div>
<div class="line"><a id="l00109" name="l00109"></a><span class="lineno"> 109</span> <span class="comment">// If current layer is the first layer set it to the last layer</span></div>
<div class="line"><a id="l00110" name="l00110"></a><span class="lineno"> 110</span> cl = cl == 1 ? ll : cl - 1;</div>
<div class="line"><a id="l00111" name="l00111"></a><span class="lineno"> 111</span> }</div>
<div class="line"><a id="l00112" name="l00112"></a><span class="lineno"> 112</span>};</div>
<div class="line"><a id="l00074" name="l00074"></a><span class="lineno"> 74</span> <span class="comment">// compute bias-corrected second raw moment estimate</span></div>
<div class="line"><a id="l00075" name="l00075"></a><span class="lineno"> 75</span> <span class="keywordtype">double</span> beta2_t = std::pow(beta2, t);</div>
<div class="line"><a id="l00076" name="l00076"></a><span class="lineno"> 76</span> </div>
<div class="line"><a id="l00077" name="l00077"></a><span class="lineno"> 77</span> <span class="keywordtype">double</span> alpha_t = alpha * (sqrt(1 - beta2_t) / (1 - beta1_t));</div>
<div class="line"><a id="l00078" name="l00078"></a><span class="lineno"> 78</span> </div>
<div class="line"><a id="l00079" name="l00079"></a><span class="lineno"> 79</span> <span class="comment">// update param</span></div>
<div class="line"><a id="l00080" name="l00080"></a><span class="lineno"> 80</span> param =</div>
<div class="line"><a id="l00081" name="l00081"></a><span class="lineno"> 81</span> param.array() - alpha_t * (m.array() / (v.array().sqrt() + epsilon));</div>
<div class="line"><a id="l00082" name="l00082"></a><span class="lineno"> 82</span> }</div>
<div class="line"><a id="l00083" name="l00083"></a><span class="lineno"> 83</span> </div>
<div class="line"><a id="l00084" name="l00084"></a><span class="lineno"> 84</span> <span class="keyword">private</span>:</div>
<div class="line"><a id="l00085" name="l00085"></a><span class="lineno"> 85</span> <span class="keywordtype">double</span> beta1;</div>
<div class="line"><a id="l00086" name="l00086"></a><span class="lineno"> 86</span> <span class="keywordtype">double</span> beta2;</div>
<div class="line"><a id="l00087" name="l00087"></a><span class="lineno"> 87</span> <span class="keywordtype">double</span> epsilon;</div>
<div class="line"><a id="l00088" name="l00088"></a><span class="lineno"> 88</span> <span class="keywordtype">int</span> t = 0;</div>
<div class="line"><a id="l00089" name="l00089"></a><span class="lineno"> 89</span> <span class="keywordtype">int</span> cl; <span class="comment">// Current layer (should be initialized to the total number of layers</span></div>
<div class="line"><a id="l00090" name="l00090"></a><span class="lineno"> 90</span> <span class="comment">// - 0)</span></div>
<div class="line"><a id="l00091" name="l00091"></a><span class="lineno"> 91</span> <span class="keywordtype">int</span> ll; <span class="comment">// Last layer (should also be initialized to numLayers - 1)</span></div>
<div class="line"><a id="l00092" name="l00092"></a><span class="lineno"> 92</span> std::vector&lt;Eigen::MatrixXd&gt; mWeights; <span class="comment">// First-moment vector for weights</span></div>
<div class="line"><a id="l00093" name="l00093"></a><span class="lineno"> 93</span> std::vector&lt;Eigen::MatrixXd&gt; vWeights; <span class="comment">// Second-moment vector for weights</span></div>
<div class="line"><a id="l00094" name="l00094"></a><span class="lineno"> 94</span> std::vector&lt;Eigen::MatrixXd&gt; mBiases; <span class="comment">// First-moment vector for biases</span></div>
<div class="line"><a id="l00095" name="l00095"></a><span class="lineno"> 95</span> std::vector&lt;Eigen::MatrixXd&gt; vBiases; <span class="comment">// Second-moment vector for biases</span></div>
<div class="line"><a id="l00096" name="l00096"></a><span class="lineno"> 96</span> </div>
<div class="line"><a id="l00097" name="l00097"></a><span class="lineno"> 97</span> <span class="keywordtype">void</span> insiderInit(<span class="keywordtype">size_t</span> numLayers)<span class="keyword"> override </span>{</div>
<div class="line"><a id="l00098" name="l00098"></a><span class="lineno"> 98</span> cl = numLayers - 1;</div>
<div class="line"><a id="l00099" name="l00099"></a><span class="lineno"> 99</span> ll = numLayers - 1;</div>
<div class="line"><a id="l00100" name="l00100"></a><span class="lineno"> 100</span> </div>
<div class="line"><a id="l00101" name="l00101"></a><span class="lineno"> 101</span> Eigen::MatrixXd dotMatrix = Eigen::MatrixXd::Zero(0, 0);</div>
<div class="line"><a id="l00102" name="l00102"></a><span class="lineno"> 102</span> </div>
<div class="line"><a id="l00103" name="l00103"></a><span class="lineno"> 103</span> <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = mWeights.size(); i &lt; numLayers; i++) {</div>
<div class="line"><a id="l00104" name="l00104"></a><span class="lineno"> 104</span> mWeights.push_back(dotMatrix);</div>
<div class="line"><a id="l00105" name="l00105"></a><span class="lineno"> 105</span> vWeights.push_back(dotMatrix);</div>
<div class="line"><a id="l00106" name="l00106"></a><span class="lineno"> 106</span> mBiases.push_back(dotMatrix);</div>
<div class="line"><a id="l00107" name="l00107"></a><span class="lineno"> 107</span> vBiases.push_back(dotMatrix);</div>
<div class="line"><a id="l00108" name="l00108"></a><span class="lineno"> 108</span> };</div>
<div class="line"><a id="l00109" name="l00109"></a><span class="lineno"> 109</span> }</div>
<div class="line"><a id="l00110" name="l00110"></a><span class="lineno"> 110</span> </div>
<div class="line"><a id="l00111" name="l00111"></a><span class="lineno"> 111</span> <span class="keywordtype">void</span> setCurrentL() {</div>
<div class="line"><a id="l00112" name="l00112"></a><span class="lineno"> 112</span> <span class="comment">// If current layer is the first layer set it to the last layer</span></div>
<div class="line"><a id="l00113" name="l00113"></a><span class="lineno"> 113</span> cl = cl == 1 ? ll : cl - 1;</div>
<div class="line"><a id="l00114" name="l00114"></a><span class="lineno"> 114</span> }</div>
<div class="line"><a id="l00115" name="l00115"></a><span class="lineno"> 115</span>};</div>
</div>
<div class="line"><a id="l00113" name="l00113"></a><span class="lineno"> 113</span>} <span class="comment">// namespace NeuralNet</span></div>
<div class="line"><a id="l00116" name="l00116"></a><span class="lineno"> 116</span>} <span class="comment">// namespace NeuralNet</span></div>
<div class="ttc" id="aclassNeuralNet_1_1Adam_html"><div class="ttname"><a href="classNeuralNet_1_1Adam.html">NeuralNet::Adam</a></div><div class="ttdef"><b>Definition</b> Adam.hpp:12</div></div>
<div class="ttc" id="aclassNeuralNet_1_1Adam_html_a13bcb5f6d78ef95b1d829e09e7e90fe4"><div class="ttname"><a href="classNeuralNet_1_1Adam.html#a13bcb5f6d78ef95b1d829e09e7e90fe4">NeuralNet::Adam::updateWeights</a></div><div class="ttdeci">void updateWeights(Eigen::MatrixXd &amp;weights, const Eigen::MatrixXd &amp;weightsGrad) override</div><div class="ttdoc">This function updates the weights passed based on the selected Optimizer and the weights gradients.</div><div class="ttdef"><b>Definition</b> Adam.hpp:34</div></div>
<div class="ttc" id="aclassNeuralNet_1_1Adam_html_a957015939ff2cb82d7a9e982d8c5e6fe"><div class="ttname"><a href="classNeuralNet_1_1Adam.html#a957015939ff2cb82d7a9e982d8c5e6fe">NeuralNet::Adam::updateBiases</a></div><div class="ttdeci">void updateBiases(Eigen::MatrixXd &amp;biases, const Eigen::MatrixXd &amp;biasesGrad) override</div><div class="ttdoc">This function updates the biases passed based based on the Optimizer and the biases gradients.</div><div class="ttdef"><b>Definition</b> Adam.hpp:39</div></div>
Expand Down

0 comments on commit 1b85deb

Please sign in to comment.