="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">custom_policies</span><span class="p">,</span>
        <span class="p">)</span>

        <span class="c1"># Replace original layer to tensor sliced (megatron) layer.</span>
        <span class="n">replacer</span><span class="o">.</span><span class="n">replace_modules</span><span class="p">()</span>

        <span class="c1"># Lazy GPU memory allocation (Only cpu tensors are loaded onto all gpus)</span>
        <span class="c1">