LoRA typo fix

Author: vpj

docs/lora/index.html

@@ -361,9 +361,7 @@ <h2>LoRA Embedding Layer</h2>
         </div>
         <div class='code'>
             <div class="highlight"><pre><span class="lineno">123</span>        <span class="k">if</span> <span class="n">alpha</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-<span class="lineno">124</span>            <span class="n">alpha</span> <span class="o">=</span> <span class="n">r</span>
-<span class="lineno">125</span>
-<span class="lineno">126</span>        <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span></pre></div>
+<span class="lineno">124</span>            <span class="n">alpha</span> <span class="o">=</span> <span class="n">r</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-22'>
@@ -375,8 +373,8 @@ <h2>LoRA Embedding Layer</h2>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">128</span>        <span class="bp">self</span><span class="o">.</span><span class="n">weight</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">((</span><span class="n">num_embeddings</span><span class="p">,</span> <span class="n">embedding_dim</span><span class="p">)))</span>
-<span class="lineno">129</span>        <span class="bp">self</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">requires_grad</span> <span class="o">=</span> <span class="kc">False</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">127</span>        <span class="bp">self</span><span class="o">.</span><span class="n">weight</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">((</span><span class="n">num_embeddings</span><span class="p">,</span> <span class="n">embedding_dim</span><span class="p">)))</span>
+<span class="lineno">128</span>        <span class="bp">self</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">requires_grad</span> <span class="o">=</span> <span class="kc">False</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-23'>
@@ -388,7 +386,7 @@ <h2>LoRA Embedding Layer</h2>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">132</span>        <span class="bp">self</span><span class="o">.</span><span class="n">scaling</span> <span class="o">=</span> <span class="n">alpha</span> <span class="o">/</span> <span class="n">r</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">131</span>        <span class="bp">self</span><span class="o">.</span><span class="n">scaling</span> <span class="o">=</span> <span class="n">alpha</span> <span class="o">/</span> <span class="n">r</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-24'>
@@ -400,7 +398,7 @@ <h2>LoRA Embedding Layer</h2>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">134</span>        <span class="bp">self</span><span class="o">.</span><span class="n">lora_a</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">((</span><span class="n">r</span><span class="p">,</span> <span class="n">num_embeddings</span><span class="p">)))</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">133</span>        <span class="bp">self</span><span class="o">.</span><span class="n">lora_a</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">((</span><span class="n">r</span><span class="p">,</span> <span class="n">num_embeddings</span><span class="p">)))</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-25'>
@@ -412,9 +410,9 @@ <h2>LoRA Embedding Layer</h2>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">136</span>        <span class="bp">self</span><span class="o">.</span><span class="n">lora_b</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">((</span><span class="n">embedding_dim</span><span class="p">,</span> <span class="n">r</span><span class="p">)))</span>
-<span class="lineno">137</span>
-<span class="lineno">138</span>        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">135</span>        <span class="bp">self</span><span class="o">.</span><span class="n">lora_b</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">((</span><span class="n">embedding_dim</span><span class="p">,</span> <span class="n">r</span><span class="p">)))</span>
+<span class="lineno">136</span>
+<span class="lineno">137</span>        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-26'>
@@ -426,7 +424,7 @@ <h2>LoRA Embedding Layer</h2>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">140</span>            <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">normal_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lora_a</span><span class="p">)</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">139</span>            <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">normal_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lora_a</span><span class="p">)</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-27'>
@@ -438,7 +436,7 @@ <h2>LoRA Embedding Layer</h2>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">142</span>            <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">zeros_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lora_b</span><span class="p">)</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">141</span>            <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">zeros_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lora_b</span><span class="p">)</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-28'>
@@ -449,7 +447,7 @@ <h2>LoRA Embedding Layer</h2>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">144</span>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">143</span>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-29'>
@@ -461,7 +459,7 @@ <h2>LoRA Embedding Layer</h2>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">146</span>        <span class="n">result</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">functional</span><span class="o">.</span><span class="n">embedding</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight</span><span class="p">)</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">145</span>        <span class="n">result</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">functional</span><span class="o">.</span><span class="n">embedding</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight</span><span class="p">)</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-30'>
@@ -473,7 +471,7 @@ <h2>LoRA Embedding Layer</h2>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">149</span>        <span class="n">result</span> <span class="o">+=</span> <span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">functional</span><span class="o">.</span><span class="n">embedding</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">lora_a</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="o">@</span> <span class="bp">self</span><span class="o">.</span><span class="n">lora_b</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaling</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">148</span>        <span class="n">result</span> <span class="o">+=</span> <span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">functional</span><span class="o">.</span><span class="n">embedding</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">lora_a</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="o">@</span> <span class="bp">self</span><span class="o">.</span><span class="n">lora_b</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaling</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-31'>
@@ -485,7 +483,7 @@ <h2>LoRA Embedding Layer</h2>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">152</span>        <span class="k">return</span> <span class="n">result</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">151</span>        <span class="k">return</span> <span class="n">result</span></pre></div>
         </div>
     </div>
     <div class='footer'>

labml_nn/lora/__init__.py

@@ -123,7 +123,6 @@ def __init__(self, num_embeddings: int, embedding_dim: int,
         if alpha is None:
             alpha = r
 
-        nn.Embedding
         # The pre-trained embedding weights $W_0^T$ (frozen)
         self.weight = nn.Parameter(torch.empty((num_embeddings, embedding_dim)))
         self.weight.requires_grad = False