an>
<span class="lineno">92</span>        <span class="s1">&#39;transformer.n_heads&#39;</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span>
<span class="lineno">93</span>        <span class="s1">&#39;transformer.n_layers&#39;</span><span class="p">:</span> <span class="mi">6</span><span class="p">,</span>
<span class="lineno">94</span>        <span class="s1">&#39;transformer.ffn.activation&#39;</span><span class="p">:</span> <span class="s1">&#39;GELU&#39;</span><span class="p