import{S as Ao,i as So,s as go,e as s,k as m,w,t as i,M as Eo,c as n,d as t,m as M,a as o,x as h,h as p,b as f,G as a,g as r,y as d,q as c,o as y,B as u,v as zo}from"../chunks/vendor-hf-doc-builder.js";import{T as _o}from"../chunks/Tip-hf-doc-builder.js";import{I as Wt}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as b}from"../chunks/CodeBlock-hf-doc-builder.js";function Lo(Bt){let T,_;return{c(){T=s("p"),_=i(`If you plan to run multiple experiments, in order to properly clear the memory between experiments, restart the Python
kernel between experiments.`)},l(U){T=n(U,"P",{});var I=o(T);_=p(I,`If you plan to run multiple experiments, in order to properly clear the memory between experiments, restart the Python
kernel between experiments.`),I.forEach(t)},m(U,I){r(U,T,I),a(T,_)},d(U){U&&t(T)}}}function Qo(Bt){let T,_,U,I,Oe,x,ol,He,rl,Vt,Te,il,Zt,Je,pl,Ft,D,xt,v,ml,Ke,Ml,fl,et,wl,hl,Dt,L,dl,X,cl,yl,Xt,q,qt,Q,ul,Ue,Tl,Jl,Yt,Y,Ot,Ie,Ul,Ht,O,Kt,je,Il,ea,H,ta,be,jl,aa,S,R,tt,K,bl,at,vl,la,G,Cl,lt,Al,Sl,sa,ee,na,k,gl,st,El,zl,oa,te,ra,ae,ia,ve,_l,pa,le,ma,$,Ma,g,P,nt,se,Ll,ot,Ql,fa,N,Rl,Ce,Gl,kl,wa,ne,ha,oe,da,Ae,$l,ca,E,W,rt,re,Pl,it,Nl,ya,Se,Wl,ua,C,ie,pt,mt,Bl,Vl,pe,Zl,Mt,Fl,xl,Dl,me,ft,wt,Xl,ql,Me,Yl,ht,Ol,Hl,Kl,fe,dt,ct,es,ts,we,as,yt,ls,ss,Ta,ge,ns,Ja,he,os,de,rs,Ua,z,B,ut,ce,is,Tt,ps,Ia,Ee,ms,ja,J,Jt,Ms,fs,Ut,ws,hs,It,ds,cs,jt,ys,us,bt,Ts,Js,vt,Us,ba,ze,Is,va,_e,js,Ca,Le,Ct,bs,Aa,V,At,vs,Cs,St,As,Sa,Qe,gt,Ss,ga,A,Et,gs,Es,Re,zs,ye,_s,Ls,zt,Qs,Ea,Ge,_t,Rs,za,ke,Lt,Gs,_a,$e,Qt,ks,La,Pe,Rt,$s,Qa,Ne,Ps,Ra,We,Gt,Ns,Ga,Be,Ws,ka,Ve,kt,Bs,$a,Ze,Vs,Pa,Fe,Z,$t,Zs,Fs,Pt,xs,Ds,Na,xe,Xs,Wa,F,qs,De,Ys,Os,Ba;return x=new Wt({}),D=new b({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwYWNjZWxlcmF0ZSUyMG52aWRpYS1tbC1weTM=",highlighted:"pip install transformers datasets accelerate nvidia-ml-py3"}}),q=new b({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQSUwQSUwQXNlcV9sZW4lMkMlMjBkYXRhc2V0X3NpemUlMjAlM0QlMjA1MTIlMkMlMjA1MTIlMEFkdW1teV9kYXRhJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIyaW5wdXRfaWRzJTIyJTNBJTIwbnAucmFuZG9tLnJhbmRpbnQoMTAwJTJDJTIwMzAwMDAlMkMlMjAoZGF0YXNldF9zaXplJTJDJTIwc2VxX2xlbikpJTJDJTBBJTIwJTIwJTIwJTIwJTIybGFiZWxzJTIyJTNBJTIwbnAucmFuZG9tLnJhbmRpbnQoMCUyQyUyMDElMkMlMjAoZGF0YXNldF9zaXplKSklMkMlMEElN0QlMEFkcyUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KGR1bW15X2RhdGEpJTBBZHMuc2V0X2Zvcm1hdCglMjJwdCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset


<span class="hljs-meta">&gt;&gt;&gt; </span>seq_len, dataset_size = <span class="hljs-number">512</span>, <span class="hljs-number">512</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dummy_data = {
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;input_ids&quot;</span>: np.random.randint(<span class="hljs-number">100</span>, <span class="hljs-number">30000</span>, (dataset_size, seq_len)),
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;labels&quot;</span>: np.random.randint(<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, (dataset_size)),
<span class="hljs-meta">... </span>}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict(dummy_data)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds.set_format(<span class="hljs-string">&quot;pt&quot;</span>)`}}),Y=new b({props:{code:"ZnJvbSUyMHB5bnZtbCUyMGltcG9ydCUyMColMEElMEElMEFkZWYlMjBwcmludF9ncHVfdXRpbGl6YXRpb24oKSUzQSUwQSUyMCUyMCUyMCUyMG52bWxJbml0KCklMEElMjAlMjAlMjAlMjBoYW5kbGUlMjAlM0QlMjBudm1sRGV2aWNlR2V0SGFuZGxlQnlJbmRleCgwKSUwQSUyMCUyMCUyMCUyMGluZm8lMjAlM0QlMjBudm1sRGV2aWNlR2V0TWVtb3J5SW5mbyhoYW5kbGUpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMkdQVSUyMG1lbW9yeSUyMG9jY3VwaWVkJTNBJTIwJTdCaW5mby51c2VkJTJGJTJGMTAyNCoqMiU3RCUyME1CLiUyMiklMEElMEElMEFkZWYlMjBwcmludF9zdW1tYXJ5KHJlc3VsdCklM0ElMEElMjAlMjAlMjAlMjBwcmludChmJTIyVGltZSUzQSUyMCU3QnJlc3VsdC5tZXRyaWNzJTVCJ3RyYWluX3J1bnRpbWUnJTVEJTNBLjJmJTdEJTIyKSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjJTYW1wbGVzJTJGc2Vjb25kJTNBJTIwJTdCcmVzdWx0Lm1ldHJpY3MlNUIndHJhaW5fc2FtcGxlc19wZXJfc2Vjb25kJyU1RCUzQS4yZiU3RCUyMiklMEElMjAlMjAlMjAlMjBwcmludF9ncHVfdXRpbGl6YXRpb24oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> pynvml <span class="hljs-keyword">import</span> *


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">print_gpu_utilization</span>():
<span class="hljs-meta">... </span>    nvmlInit()
<span class="hljs-meta">... </span>    handle = nvmlDeviceGetHandleByIndex(<span class="hljs-number">0</span>)
<span class="hljs-meta">... </span>    info = nvmlDeviceGetMemoryInfo(handle)
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;GPU memory occupied: <span class="hljs-subst">{info.used//<span class="hljs-number">1024</span>**<span class="hljs-number">2</span>}</span> MB.&quot;</span>)


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">print_summary</span>(<span class="hljs-params">result</span>):
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Time: <span class="hljs-subst">{result.metrics[<span class="hljs-string">&#x27;train_runtime&#x27;</span>]:<span class="hljs-number">.2</span>f}</span>&quot;</span>)
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Samples/second: <span class="hljs-subst">{result.metrics[<span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>]:<span class="hljs-number">.2</span>f}</span>&quot;</span>)
<span class="hljs-meta">... </span>    print_gpu_utilization()`}}),O=new b({props:{code:"cHJpbnRfZ3B1X3V0aWxpemF0aW9uKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>print_gpu_utilization()
GPU memory occupied: <span class="hljs-number">0</span> MB.`}}),H=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEElMEF0b3JjaC5vbmVzKCgxJTJDJTIwMSkpLnRvKCUyMmN1ZGElMjIpJTBBcHJpbnRfZ3B1X3V0aWxpemF0aW9uKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch


<span class="hljs-meta">&gt;&gt;&gt; </span>torch.ones((<span class="hljs-number">1</span>, <span class="hljs-number">1</span>)).to(<span class="hljs-string">&quot;cuda&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>print_gpu_utilization()
GPU memory occupied: <span class="hljs-number">1343</span> MB.`}}),K=new Wt({}),ee=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmJlcnQtbGFyZ2UtdW5jYXNlZCUyMikudG8oJTIyY3VkYSUyMiklMEFwcmludF9ncHVfdXRpbGl6YXRpb24oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification


<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-large-uncased&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>print_gpu_utilization()
GPU memory occupied: <span class="hljs-number">2631</span> MB.`}}),te=new b({props:{code:"bnZpZGlhLXNtaQ==",highlighted:"nvidia-smi"}}),ae=new b({props:{code:"VHVlJTIwSmFuJTIwMTElMjAwOCUzQTU4JTNBMDUlMjAyMDIyJTBBJTJCLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0lMkIlMEElN0MlMjBOVklESUEtU01JJTIwNDYwLjkxLjAzJTIwJTIwJTIwJTIwRHJpdmVyJTIwVmVyc2lvbiUzQSUyMDQ2MC45MS4wMyUyMCUyMCUyMCUyMENVREElMjBWZXJzaW9uJTNBJTIwMTEuMiUyMCUyMCUyMCUyMCUyMCU3QyUwQSU3Qy0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0lMkItLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTJCLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSUyQiUwQSU3QyUyMEdQVSUyMCUyME5hbWUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBQZXJzaXN0ZW5jZS1NJTdDJTIwQnVzLUlkJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwRGlzcC5BJTIwJTdDJTIwVm9sYXRpbGUlMjBVbmNvcnIuJTIwRUNDJTIwJTdDJTBBJTdDJTIwRmFuJTIwJTIwVGVtcCUyMCUyMFBlcmYlMjAlMjBQd3IlM0FVc2FnZSUyRkNhcCU3QyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME1lbW9yeS1Vc2FnZSUyMCU3QyUyMEdQVS1VdGlsJTIwJTIwQ29tcHV0ZSUyME0uJTIwJTdDJTBBJTdDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTUlHJTIwTS4lMjAlN0MlMEElN0MlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMkIlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMkIlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlN0MlMEElN0MlMjAlMjAlMjAwJTIwJTIwVGVzbGElMjBWMTAwLVNYTTIuLi4lMjAlMjBPbiUyMCUyMCUyMCU3QyUyMDAwMDAwMDAwJTNBMDAlM0EwNC4wJTIwT2ZmJTIwJTdDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwMCUyMCU3QyUwQSU3QyUyME4lMkZBJTIwJTIwJTIwMzdDJTIwJTIwJTIwJTIwUDAlMjAlMjAlMjAlMjAzOVclMjAlMkYlMjAzMDBXJTIwJTdDJTIwJTIwJTIwMjYzMU1pQiUyMCUyRiUyMDE2MTYwTWlCJTIwJTdDJTIwJTIwJTIwJTIwJTIwJTIwMCUyNSUyMCUyMCUyMCUyMCUyMCUyMERlZmF1bHQlMjAlN0MlMEElN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBOJTJGQSUyMCU3QyUwQSUyQi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0lMkItLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTJCLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSUyQiUwQSUwQSUyQi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTJCJTBBJTdDJTIwUHJvY2Vzc2VzJTNBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdDJTBBJTdDJTIwJTIwR1BVJTIwJTIwJTIwR0klMjAlMjAlMjBDSSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFBJRCUyMCUyMCUyMFR5cGUlMjAlMjAlMjBQcm9jZXNzJTIwbmFtZSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEdQVSUyME1lbW9yeSUyMCU3QyUwQSU3QyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMElEJTIwJTIwJTIwSUQlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBVc2FnZSUyMCUyMCUyMCUyMCUyMCUyMCU3QyUwQSU3QyUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCU3QyUwQSU3QyUyMCUyMCUyMCUyMDAlMjAlMjAlMjBOJTJGQSUyMCUyME4lMkZBJTIwJTIwJTIwJTIwJTIwJTIwMzcyMSUyMCUyMCUyMCUyMCUyMCUyMEMlMjAlMjAlMjAuLi5udnMlMkZjb2RlcGFycm90JTJGYmluJTJGcHl0aG9uJTIwJTIwJTIwJTIwJTIwMjYyOU1pQiUyMCU3QyUwQSUyQi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTJC",highlighted:`Tue Jan 11 08:58:05 2022
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 460.91.03    Driver Version: 460.91.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla V100-SXM2...  On   | 00000000:00:04.0 Off |                    0 |
| N/A   37C    P0    39W / 300W |   2631MiB / 16160MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A      3721      C   ...nvs/codeparrot/bin/python     2629MiB |
+-----------------------------------------------------------------------------+`}}),le=new b({props:{code:"ZGVmYXVsdF9hcmdzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIyb3V0cHV0X2RpciUyMiUzQSUyMCUyMnRtcCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmV2YWx1YXRpb25fc3RyYXRlZ3klMjIlM0ElMjAlMjJzdGVwcyUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMm51bV90cmFpbl9lcG9jaHMlMjIlM0ElMjAxJTJDJTBBJTIwJTIwJTIwJTIwJTIybG9nX2xldmVsJTIyJTNBJTIwJTIyZXJyb3IlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJyZXBvcnRfdG8lMjIlM0ElMjAlMjJub25lJTIyJTJDJTBBJTdE",highlighted:`default_args = {
    <span class="hljs-string">&quot;output_dir&quot;</span>: <span class="hljs-string">&quot;tmp&quot;</span>,
    <span class="hljs-string">&quot;evaluation_strategy&quot;</span>: <span class="hljs-string">&quot;steps&quot;</span>,
    <span class="hljs-string">&quot;num_train_epochs&quot;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&quot;log_level&quot;</span>: <span class="hljs-string">&quot;error&quot;</span>,
    <span class="hljs-string">&quot;report_to&quot;</span>: <span class="hljs-string">&quot;none&quot;</span>,
}`}}),$=new _o({props:{$$slots:{default:[Lo]},$$scope:{ctx:Bt}}}),se=new Wt({}),ne=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTJDJTIwVHJhaW5lciUyQyUyMGxvZ2dpbmclMEElMEFsb2dnaW5nLnNldF92ZXJib3NpdHlfZXJyb3IoKSUwQSUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyhwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0Q0JTJDJTIwKipkZWZhdWx0X2FyZ3MpJTBBdHJhaW5lciUyMCUzRCUyMFRyYWluZXIobW9kZWwlM0Rtb2RlbCUyQyUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTIwdHJhaW5fZGF0YXNldCUzRGRzKSUwQXJlc3VsdCUyMCUzRCUyMHRyYWluZXIudHJhaW4oKSUwQXByaW50X3N1bW1hcnkocmVzdWx0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, Trainer, logging

<span class="hljs-meta">&gt;&gt;&gt; </span>logging.set_verbosity_error()


<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(per_device_train_batch_size=<span class="hljs-number">4</span>, **default_args)
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(model=model, args=training_args, train_dataset=ds)
<span class="hljs-meta">&gt;&gt;&gt; </span>result = trainer.train()
<span class="hljs-meta">&gt;&gt;&gt; </span>print_summary(result)`}}),oe=new b({props:{code:"VGltZSUzQSUyMDU3LjgyJTBBU2FtcGxlcyUyRnNlY29uZCUzQSUyMDguODYlMEFHUFUlMjBtZW1vcnklMjBvY2N1cGllZCUzQSUyMDE0OTQ5JTIwTUIu",highlighted:`<span class="hljs-keyword">Time:</span> 57.82
Samples/second: 8.86
GPU memory occupied: 14949 MB.`}}),re=new Wt({}),ce=new Wt({}),{c(){T=s("meta"),_=m(),U=s("h1"),I=s("a"),Oe=s("span"),w(x.$$.fragment),ol=m(),He=s("span"),rl=i("Model training anatomy"),Vt=m(),Te=s("p"),il=i(`To understand performance optimization techniques that one can apply to improve efficiency of model training
speed and memory utilization, it\u2019s helpful to get familiar with how GPU is utilized during training, and how compute
intensity varies depending on an operation performed.`),Zt=m(),Je=s("p"),pl=i(`Let\u2019s start by exploring a motivating example of GPU utilization and the training run of a model. For the demonstration,
we\u2019ll need to install a few libraries:`),Ft=m(),w(D.$$.fragment),xt=m(),v=s("p"),ml=i("The "),Ke=s("code"),Ml=i("nvidia-ml-py3"),fl=i(` library allows us to monitor the memory usage of the models from within Python. You might be familiar
with the `),et=s("code"),wl=i("nvidia-smi"),hl=i(" command in the terminal - this library allows to access the same information in Python directly."),Dt=m(),L=s("p"),dl=i(`Then, we create some dummy data: random token IDs between 100 and 30000 and binary labels for a classifier.
In total, we get 512 sequences each with length 512 and store them in a `),X=s("a"),cl=i("Dataset"),yl=i(" with PyTorch format."),Xt=m(),w(q.$$.fragment),qt=m(),Q=s("p"),ul=i("To print summary statistics for the GPU utilization and the training run with the "),Ue=s("a"),Tl=i("Trainer"),Jl=i(" we define two helper functions:"),Yt=m(),w(Y.$$.fragment),Ot=m(),Ie=s("p"),Ul=i("Let\u2019s verify that we start with a free GPU memory:"),Ht=m(),w(O.$$.fragment),Kt=m(),je=s("p"),Il=i(`That looks good: the GPU memory is not occupied as we would expect before we load any models. If that\u2019s not the case on
your machine make sure to stop all processes that are using GPU memory. However, not all free GPU memory can be used by
the user. When a model is loaded to the GPU the kernels are also loaded, which can take up 1-2GB of memory. To see how
much it is we load a tiny tensor into the GPU which triggers the kernels to be loaded as well.`),ea=m(),w(H.$$.fragment),ta=m(),be=s("p"),jl=i("We see that the kernels alone take up 1.3GB of GPU memory. Now let\u2019s see how much space the model uses."),aa=m(),S=s("h2"),R=s("a"),tt=s("span"),w(K.$$.fragment),bl=m(),at=s("span"),vl=i("Load Model"),la=m(),G=s("p"),Cl=i("First, we load the "),lt=s("code"),Al=i("bert-large-uncased"),Sl=i(` model. We load the model weights directly to the GPU so that we can check
how much space just the weights use.`),sa=m(),w(ee.$$.fragment),na=m(),k=s("p"),gl=i(`We can see that the model weights alone take up 1.3 GB of GPU memory. The exact number depends on the specific
GPU you are using. Note that on newer GPUs a model can sometimes take up more space since the weights are loaded in an
optimized fashion that speeds up the usage of the model. Now we can also quickly check if we get the same result
as with `),st=s("code"),El=i("nvidia-smi"),zl=i(" CLI:"),oa=m(),w(te.$$.fragment),ra=m(),w(ae.$$.fragment),ia=m(),ve=s("p"),_l=i(`We get the same number as before and you can also see that we are using a V100 GPU with 16GB of memory. So now we can
start training the model and see how the GPU memory consumption changes. First, we set up a few standard training
arguments:`),pa=m(),w(le.$$.fragment),ma=m(),w($.$$.fragment),Ma=m(),g=s("h2"),P=s("a"),nt=s("span"),w(se.$$.fragment),Ll=m(),ot=s("span"),Ql=i("Memory utilization at vanilla training"),fa=m(),N=s("p"),Rl=i("Let\u2019s use the "),Ce=s("a"),Gl=i("Trainer"),kl=i(" and train the model without using any GPU performance optimization techniques and a batch size of 4:"),wa=m(),w(ne.$$.fragment),ha=m(),w(oe.$$.fragment),da=m(),Ae=s("p"),$l=i(`We see that already a relatively small batch size almost fills up our GPU\u2019s entire memory. However, a larger batch size
can often result in faster model convergence or better end performance. So ideally we want to tune the batch size to our
model\u2019s needs and not to the GPU limitations. What\u2019s interesting is that we use much more memory than the size of the model.
To understand a bit better why this is the case let\u2019s have a look at a model\u2019s operations and memory needs.`),ca=m(),E=s("h2"),W=s("a"),rt=s("span"),w(re.$$.fragment),Pl=m(),it=s("span"),Nl=i("Anatomy of Model's Operations"),ya=m(),Se=s("p"),Wl=i("Transformers architecture includes 3 main groups of operations grouped below by compute-intensity."),ua=m(),C=s("ol"),ie=s("li"),pt=s("p"),mt=s("strong"),Bl=i("Tensor Contractions"),Vl=m(),pe=s("p"),Zl=i("Linear layers and components of Multi-Head Attention all do batched "),Mt=s("strong"),Fl=i("matrix-matrix multiplications"),xl=i(". These operations are the most compute-intensive part of training a transformer."),Dl=m(),me=s("li"),ft=s("p"),wt=s("strong"),Xl=i("Statistical Normalizations"),ql=m(),Me=s("p"),Yl=i("Softmax and layer normalization are less compute-intensive than tensor contractions, and involve one or more "),ht=s("strong"),Ol=i("reduction operations"),Hl=i(", the result of which is then applied via a map."),Kl=m(),fe=s("li"),dt=s("p"),ct=s("strong"),es=i("Element-wise Operators"),ts=m(),we=s("p"),as=i("These are the remaining operators: "),yt=s("strong"),ls=i("biases, dropout, activations, and residual connections"),ss=i(". These are the least compute-intensive operations."),Ta=m(),ge=s("p"),ns=i("This knowledge can be helpful to know when analyzing performance bottlenecks."),Ja=m(),he=s("p"),os=i("This summary is derived from "),de=s("a"),rs=i("Data Movement Is All You Need: A Case Study on Optimizing Transformers 2020"),Ua=m(),z=s("h2"),B=s("a"),ut=s("span"),w(ce.$$.fragment),is=m(),Tt=s("span"),ps=i("Anatomy of Model's Memory"),Ia=m(),Ee=s("p"),ms=i(`We\u2019ve seen that training the model uses much more memory than just putting the model on the GPU. This is because there
are many components during training that use GPU memory. The components on GPU memory are the following:`),ja=m(),J=s("ol"),Jt=s("li"),Ms=i("model weights"),fs=m(),Ut=s("li"),ws=i("optimizer states"),hs=m(),It=s("li"),ds=i("gradients"),cs=m(),jt=s("li"),ys=i("forward activations saved for gradient computation"),us=m(),bt=s("li"),Ts=i("temporary buffers"),Js=m(),vt=s("li"),Us=i("functionality-specific memory"),ba=m(),ze=s("p"),Is=i(`A typical model trained in mixed precision with AdamW requires 18 bytes per model parameter plus activation memory. For
inference there are no optimizer states and gradients, so we can subtract those. And thus we end up with 6 bytes per
model parameter for mixed precision inference, plus activation memory.`),va=m(),_e=s("p"),js=i("Let\u2019s look at the details."),Ca=m(),Le=s("p"),Ct=s("strong"),bs=i("Model Weights:"),Aa=m(),V=s("ul"),At=s("li"),vs=i("4 bytes * number of parameters for fp32 training"),Cs=m(),St=s("li"),As=i("6 bytes * number of parameters for mixed precision training (maintains a model in fp32 and one in fp16 in memory)"),Sa=m(),Qe=s("p"),gt=s("strong"),Ss=i("Optimizer States:"),ga=m(),A=s("ul"),Et=s("li"),gs=i("8 bytes * number of parameters for normal AdamW (maintains 2 states)"),Es=m(),Re=s("li"),zs=i("2 bytes * number of parameters for 8-bit AdamW optimizers like "),ye=s("a"),_s=i("bitsandbytes"),Ls=m(),zt=s("li"),Qs=i("4 bytes * number of parameters for optimizers like SGD with momentum (maintains only 1 state)"),Ea=m(),Ge=s("p"),_t=s("strong"),Rs=i("Gradients"),za=m(),ke=s("ul"),Lt=s("li"),Gs=i("4 bytes * number of parameters for either fp32 or mixed precision training (gradients are always kept in fp32)"),_a=m(),$e=s("p"),Qt=s("strong"),ks=i("Forward Activations"),La=m(),Pe=s("ul"),Rt=s("li"),$s=i("size depends on many factors, the key ones being sequence length, hidden size and batch size."),Qa=m(),Ne=s("p"),Ps=i(`There are the input and output that are being passed and returned by the forward and the backward functions and the
forward activations saved for gradient computation.`),Ra=m(),We=s("p"),Gt=s("strong"),Ns=i("Temporary Memory"),Ga=m(),Be=s("p"),Ws=i(`Additionally, there are all kinds of temporary variables which get released once the calculation is done, but in the
moment these could require additional memory and could push to OOM. Therefore, when coding it\u2019s crucial to think
strategically about such temporary variables and sometimes to explicitly free those as soon as they are no longer needed.`),ka=m(),Ve=s("p"),kt=s("strong"),Bs=i("Functionality-specific memory"),$a=m(),Ze=s("p"),Vs=i(`Then, your software could have special memory needs. For example, when generating text using beam search, the software
needs to maintain multiple copies of inputs and outputs.`),Pa=m(),Fe=s("p"),Z=s("strong"),$t=s("code"),Zs=i("forward"),Fs=i(" vs "),Pt=s("code"),xs=i("backward"),Ds=i(" Execution Speed"),Na=m(),xe=s("p"),Xs=i(`For convolutions and linear layers there are 2x flops in the backward compared to the forward, which generally translates
into ~2x slower (sometimes more, because sizes in the backward tend to be more awkward). Activations are usually
bandwidth-limited, and it\u2019s typical for an activation to have to read more data in the backward than in the forward
(e.g. activation forward reads once, writes once, activation backward reads twice, gradOutput and output of the forward,
and writes once, gradInput).`),Wa=m(),F=s("p"),qs=i(`As you can see, there are potentially a few places where we could save GPU memory or speed up operations.
Now that you understand what affects GPU utilization and computation speed, refer to
the `),De=s("a"),Ys=i("Methods and tools for efficient training on a single GPU"),Os=i(` documentation page to learn about
performance optimization techniques.`),this.h()},l(e){const l=Eo('[data-svelte="svelte-1phssyn"]',document.head);T=n(l,"META",{name:!0,content:!0}),l.forEach(t),_=M(e),U=n(e,"H1",{class:!0});var ue=o(U);I=n(ue,"A",{id:!0,class:!0,href:!0});var en=o(I);Oe=n(en,"SPAN",{});var tn=o(Oe);h(x.$$.fragment,tn),tn.forEach(t),en.forEach(t),ol=M(ue),He=n(ue,"SPAN",{});var an=o(He);rl=p(an,"Model training anatomy"),an.forEach(t),ue.forEach(t),Vt=M(e),Te=n(e,"P",{});var ln=o(Te);il=p(ln,`To understand performance optimization techniques that one can apply to improve efficiency of model training
speed and memory utilization, it\u2019s helpful to get familiar with how GPU is utilized during training, and how compute
intensity varies depending on an operation performed.`),ln.forEach(t),Zt=M(e),Je=n(e,"P",{});var sn=o(Je);pl=p(sn,`Let\u2019s start by exploring a motivating example of GPU utilization and the training run of a model. For the demonstration,
we\u2019ll need to install a few libraries:`),sn.forEach(t),Ft=M(e),h(D.$$.fragment,e),xt=M(e),v=n(e,"P",{});var Xe=o(v);ml=p(Xe,"The "),Ke=n(Xe,"CODE",{});var nn=o(Ke);Ml=p(nn,"nvidia-ml-py3"),nn.forEach(t),fl=p(Xe,` library allows us to monitor the memory usage of the models from within Python. You might be familiar
with the `),et=n(Xe,"CODE",{});var on=o(et);wl=p(on,"nvidia-smi"),on.forEach(t),hl=p(Xe," command in the terminal - this library allows to access the same information in Python directly."),Xe.forEach(t),Dt=M(e),L=n(e,"P",{});var Va=o(L);dl=p(Va,`Then, we create some dummy data: random token IDs between 100 and 30000 and binary labels for a classifier.
In total, we get 512 sequences each with length 512 and store them in a `),X=n(Va,"A",{href:!0,rel:!0});var rn=o(X);cl=p(rn,"Dataset"),rn.forEach(t),yl=p(Va," with PyTorch format."),Va.forEach(t),Xt=M(e),h(q.$$.fragment,e),qt=M(e),Q=n(e,"P",{});var Za=o(Q);ul=p(Za,"To print summary statistics for the GPU utilization and the training run with the "),Ue=n(Za,"A",{href:!0});var pn=o(Ue);Tl=p(pn,"Trainer"),pn.forEach(t),Jl=p(Za," we define two helper functions:"),Za.forEach(t),Yt=M(e),h(Y.$$.fragment,e),Ot=M(e),Ie=n(e,"P",{});var mn=o(Ie);Ul=p(mn,"Let\u2019s verify that we start with a free GPU memory:"),mn.forEach(t),Ht=M(e),h(O.$$.fragment,e),Kt=M(e),je=n(e,"P",{});var Mn=o(je);Il=p(Mn,`That looks good: the GPU memory is not occupied as we would expect before we load any models. If that\u2019s not the case on
your machine make sure to stop all processes that are using GPU memory. However, not all free GPU memory can be used by
the user. When a model is loaded to the GPU the kernels are also loaded, which can take up 1-2GB of memory. To see how
much it is we load a tiny tensor into the GPU which triggers the kernels to be loaded as well.`),Mn.forEach(t),ea=M(e),h(H.$$.fragment,e),ta=M(e),be=n(e,"P",{});var fn=o(be);jl=p(fn,"We see that the kernels alone take up 1.3GB of GPU memory. Now let\u2019s see how much space the model uses."),fn.forEach(t),aa=M(e),S=n(e,"H2",{class:!0});var Fa=o(S);R=n(Fa,"A",{id:!0,class:!0,href:!0});var wn=o(R);tt=n(wn,"SPAN",{});var hn=o(tt);h(K.$$.fragment,hn),hn.forEach(t),wn.forEach(t),bl=M(Fa),at=n(Fa,"SPAN",{});var dn=o(at);vl=p(dn,"Load Model"),dn.forEach(t),Fa.forEach(t),la=M(e),G=n(e,"P",{});var xa=o(G);Cl=p(xa,"First, we load the "),lt=n(xa,"CODE",{});var cn=o(lt);Al=p(cn,"bert-large-uncased"),cn.forEach(t),Sl=p(xa,` model. We load the model weights directly to the GPU so that we can check
how much space just the weights use.`),xa.forEach(t),sa=M(e),h(ee.$$.fragment,e),na=M(e),k=n(e,"P",{});var Da=o(k);gl=p(Da,`We can see that the model weights alone take up 1.3 GB of GPU memory. The exact number depends on the specific
GPU you are using. Note that on newer GPUs a model can sometimes take up more space since the weights are loaded in an
optimized fashion that speeds up the usage of the model. Now we can also quickly check if we get the same result
as with `),st=n(Da,"CODE",{});var yn=o(st);El=p(yn,"nvidia-smi"),yn.forEach(t),zl=p(Da," CLI:"),Da.forEach(t),oa=M(e),h(te.$$.fragment,e),ra=M(e),h(ae.$$.fragment,e),ia=M(e),ve=n(e,"P",{});var un=o(ve);_l=p(un,`We get the same number as before and you can also see that we are using a V100 GPU with 16GB of memory. So now we can
start training the model and see how the GPU memory consumption changes. First, we set up a few standard training
arguments:`),un.forEach(t),pa=M(e),h(le.$$.fragment,e),ma=M(e),h($.$$.fragment,e),Ma=M(e),g=n(e,"H2",{class:!0});var Xa=o(g);P=n(Xa,"A",{id:!0,class:!0,href:!0});var Tn=o(P);nt=n(Tn,"SPAN",{});var Jn=o(nt);h(se.$$.fragment,Jn),Jn.forEach(t),Tn.forEach(t),Ll=M(Xa),ot=n(Xa,"SPAN",{});var Un=o(ot);Ql=p(Un,"Memory utilization at vanilla training"),Un.forEach(t),Xa.forEach(t),fa=M(e),N=n(e,"P",{});var qa=o(N);Rl=p(qa,"Let\u2019s use the "),Ce=n(qa,"A",{href:!0});var In=o(Ce);Gl=p(In,"Trainer"),In.forEach(t),kl=p(qa," and train the model without using any GPU performance optimization techniques and a batch size of 4:"),qa.forEach(t),wa=M(e),h(ne.$$.fragment,e),ha=M(e),h(oe.$$.fragment,e),da=M(e),Ae=n(e,"P",{});var jn=o(Ae);$l=p(jn,`We see that already a relatively small batch size almost fills up our GPU\u2019s entire memory. However, a larger batch size
can often result in faster model convergence or better end performance. So ideally we want to tune the batch size to our
model\u2019s needs and not to the GPU limitations. What\u2019s interesting is that we use much more memory than the size of the model.
To understand a bit better why this is the case let\u2019s have a look at a model\u2019s operations and memory needs.`),jn.forEach(t),ca=M(e),E=n(e,"H2",{class:!0});var Ya=o(E);W=n(Ya,"A",{id:!0,class:!0,href:!0});var bn=o(W);rt=n(bn,"SPAN",{});var vn=o(rt);h(re.$$.fragment,vn),vn.forEach(t),bn.forEach(t),Pl=M(Ya),it=n(Ya,"SPAN",{});var Cn=o(it);Nl=p(Cn,"Anatomy of Model's Operations"),Cn.forEach(t),Ya.forEach(t),ya=M(e),Se=n(e,"P",{});var An=o(Se);Wl=p(An,"Transformers architecture includes 3 main groups of operations grouped below by compute-intensity."),An.forEach(t),ua=M(e),C=n(e,"OL",{});var qe=o(C);ie=n(qe,"LI",{});var Oa=o(ie);pt=n(Oa,"P",{});var Sn=o(pt);mt=n(Sn,"STRONG",{});var gn=o(mt);Bl=p(gn,"Tensor Contractions"),gn.forEach(t),Sn.forEach(t),Vl=M(Oa),pe=n(Oa,"P",{});var Ha=o(pe);Zl=p(Ha,"Linear layers and components of Multi-Head Attention all do batched "),Mt=n(Ha,"STRONG",{});var En=o(Mt);Fl=p(En,"matrix-matrix multiplications"),En.forEach(t),xl=p(Ha,". These operations are the most compute-intensive part of training a transformer."),Ha.forEach(t),Oa.forEach(t),Dl=M(qe),me=n(qe,"LI",{});var Ka=o(me);ft=n(Ka,"P",{});var zn=o(ft);wt=n(zn,"STRONG",{});var _n=o(wt);Xl=p(_n,"Statistical Normalizations"),_n.forEach(t),zn.forEach(t),ql=M(Ka),Me=n(Ka,"P",{});var el=o(Me);Yl=p(el,"Softmax and layer normalization are less compute-intensive than tensor contractions, and involve one or more "),ht=n(el,"STRONG",{});var Ln=o(ht);Ol=p(Ln,"reduction operations"),Ln.forEach(t),Hl=p(el,", the result of which is then applied via a map."),el.forEach(t),Ka.forEach(t),Kl=M(qe),fe=n(qe,"LI",{});var tl=o(fe);dt=n(tl,"P",{});var Qn=o(dt);ct=n(Qn,"STRONG",{});var Rn=o(ct);es=p(Rn,"Element-wise Operators"),Rn.forEach(t),Qn.forEach(t),ts=M(tl),we=n(tl,"P",{});var al=o(we);as=p(al,"These are the remaining operators: "),yt=n(al,"STRONG",{});var Gn=o(yt);ls=p(Gn,"biases, dropout, activations, and residual connections"),Gn.forEach(t),ss=p(al,". These are the least compute-intensive operations."),al.forEach(t),tl.forEach(t),qe.forEach(t),Ta=M(e),ge=n(e,"P",{});var kn=o(ge);ns=p(kn,"This knowledge can be helpful to know when analyzing performance bottlenecks."),kn.forEach(t),Ja=M(e),he=n(e,"P",{});var Hs=o(he);os=p(Hs,"This summary is derived from "),de=n(Hs,"A",{href:!0,rel:!0});var $n=o(de);rs=p($n,"Data Movement Is All You Need: A Case Study on Optimizing Transformers 2020"),$n.forEach(t),Hs.forEach(t),Ua=M(e),z=n(e,"H2",{class:!0});var ll=o(z);B=n(ll,"A",{id:!0,class:!0,href:!0});var Pn=o(B);ut=n(Pn,"SPAN",{});var Nn=o(ut);h(ce.$$.fragment,Nn),Nn.forEach(t),Pn.forEach(t),is=M(ll),Tt=n(ll,"SPAN",{});var Wn=o(Tt);ps=p(Wn,"Anatomy of Model's Memory"),Wn.forEach(t),ll.forEach(t),Ia=M(e),Ee=n(e,"P",{});var Bn=o(Ee);ms=p(Bn,`We\u2019ve seen that training the model uses much more memory than just putting the model on the GPU. This is because there
are many components during training that use GPU memory. The components on GPU memory are the following:`),Bn.forEach(t),ja=M(e),J=n(e,"OL",{});var j=o(J);Jt=n(j,"LI",{});var Vn=o(Jt);Ms=p(Vn,"model weights"),Vn.forEach(t),fs=M(j),Ut=n(j,"LI",{});var Zn=o(Ut);ws=p(Zn,"optimizer states"),Zn.forEach(t),hs=M(j),It=n(j,"LI",{});var Fn=o(It);ds=p(Fn,"gradients"),Fn.forEach(t),cs=M(j),jt=n(j,"LI",{});var xn=o(jt);ys=p(xn,"forward activations saved for gradient computation"),xn.forEach(t),us=M(j),bt=n(j,"LI",{});var Dn=o(bt);Ts=p(Dn,"temporary buffers"),Dn.forEach(t),Js=M(j),vt=n(j,"LI",{});var Xn=o(vt);Us=p(Xn,"functionality-specific memory"),Xn.forEach(t),j.forEach(t),ba=M(e),ze=n(e,"P",{});var qn=o(ze);Is=p(qn,`A typical model trained in mixed precision with AdamW requires 18 bytes per model parameter plus activation memory. For
inference there are no optimizer states and gradients, so we can subtract those. And thus we end up with 6 bytes per
model parameter for mixed precision inference, plus activation memory.`),qn.forEach(t),va=M(e),_e=n(e,"P",{});var Yn=o(_e);js=p(Yn,"Let\u2019s look at the details."),Yn.forEach(t),Ca=M(e),Le=n(e,"P",{});var On=o(Le);Ct=n(On,"STRONG",{});var Hn=o(Ct);bs=p(Hn,"Model Weights:"),Hn.forEach(t),On.forEach(t),Aa=M(e),V=n(e,"UL",{});var sl=o(V);At=n(sl,"LI",{});var Kn=o(At);vs=p(Kn,"4 bytes * number of parameters for fp32 training"),Kn.forEach(t),Cs=M(sl),St=n(sl,"LI",{});var eo=o(St);As=p(eo,"6 bytes * number of parameters for mixed precision training (maintains a model in fp32 and one in fp16 in memory)"),eo.forEach(t),sl.forEach(t),Sa=M(e),Qe=n(e,"P",{});var to=o(Qe);gt=n(to,"STRONG",{});var ao=o(gt);Ss=p(ao,"Optimizer States:"),ao.forEach(t),to.forEach(t),ga=M(e),A=n(e,"UL",{});var Ye=o(A);Et=n(Ye,"LI",{});var lo=o(Et);gs=p(lo,"8 bytes * number of parameters for normal AdamW (maintains 2 states)"),lo.forEach(t),Es=M(Ye),Re=n(Ye,"LI",{});var Ks=o(Re);zs=p(Ks,"2 bytes * number of parameters for 8-bit AdamW optimizers like "),ye=n(Ks,"A",{href:!0,rel:!0});var so=o(ye);_s=p(so,"bitsandbytes"),so.forEach(t),Ks.forEach(t),Ls=M(Ye),zt=n(Ye,"LI",{});var no=o(zt);Qs=p(no,"4 bytes * number of parameters for optimizers like SGD with momentum (maintains only 1 state)"),no.forEach(t),Ye.forEach(t),Ea=M(e),Ge=n(e,"P",{});var oo=o(Ge);_t=n(oo,"STRONG",{});var ro=o(_t);Rs=p(ro,"Gradients"),ro.forEach(t),oo.forEach(t),za=M(e),ke=n(e,"UL",{});var io=o(ke);Lt=n(io,"LI",{});var po=o(Lt);Gs=p(po,"4 bytes * number of parameters for either fp32 or mixed precision training (gradients are always kept in fp32)"),po.forEach(t),io.forEach(t),_a=M(e),$e=n(e,"P",{});var mo=o($e);Qt=n(mo,"STRONG",{});var Mo=o(Qt);ks=p(Mo,"Forward Activations"),Mo.forEach(t),mo.forEach(t),La=M(e),Pe=n(e,"UL",{});var fo=o(Pe);Rt=n(fo,"LI",{});var wo=o(Rt);$s=p(wo,"size depends on many factors, the key ones being sequence length, hidden size and batch size."),wo.forEach(t),fo.forEach(t),Qa=M(e),Ne=n(e,"P",{});var ho=o(Ne);Ps=p(ho,`There are the input and output that are being passed and returned by the forward and the backward functions and the
forward activations saved for gradient computation.`),ho.forEach(t),Ra=M(e),We=n(e,"P",{});var co=o(We);Gt=n(co,"STRONG",{});var yo=o(Gt);Ns=p(yo,"Temporary Memory"),yo.forEach(t),co.forEach(t),Ga=M(e),Be=n(e,"P",{});var uo=o(Be);Ws=p(uo,`Additionally, there are all kinds of temporary variables which get released once the calculation is done, but in the
moment these could require additional memory and could push to OOM. Therefore, when coding it\u2019s crucial to think
strategically about such temporary variables and sometimes to explicitly free those as soon as they are no longer needed.`),uo.forEach(t),ka=M(e),Ve=n(e,"P",{});var To=o(Ve);kt=n(To,"STRONG",{});var Jo=o(kt);Bs=p(Jo,"Functionality-specific memory"),Jo.forEach(t),To.forEach(t),$a=M(e),Ze=n(e,"P",{});var Uo=o(Ze);Vs=p(Uo,`Then, your software could have special memory needs. For example, when generating text using beam search, the software
needs to maintain multiple copies of inputs and outputs.`),Uo.forEach(t),Pa=M(e),Fe=n(e,"P",{});var Io=o(Fe);Z=n(Io,"STRONG",{});var Nt=o(Z);$t=n(Nt,"CODE",{});var jo=o($t);Zs=p(jo,"forward"),jo.forEach(t),Fs=p(Nt," vs "),Pt=n(Nt,"CODE",{});var bo=o(Pt);xs=p(bo,"backward"),bo.forEach(t),Ds=p(Nt," Execution Speed"),Nt.forEach(t),Io.forEach(t),Na=M(e),xe=n(e,"P",{});var vo=o(xe);Xs=p(vo,`For convolutions and linear layers there are 2x flops in the backward compared to the forward, which generally translates
into ~2x slower (sometimes more, because sizes in the backward tend to be more awkward). Activations are usually
bandwidth-limited, and it\u2019s typical for an activation to have to read more data in the backward than in the forward
(e.g. activation forward reads once, writes once, activation backward reads twice, gradOutput and output of the forward,
and writes once, gradInput).`),vo.forEach(t),Wa=M(e),F=n(e,"P",{});var nl=o(F);qs=p(nl,`As you can see, there are potentially a few places where we could save GPU memory or speed up operations.
Now that you understand what affects GPU utilization and computation speed, refer to
the `),De=n(nl,"A",{href:!0});var Co=o(De);Ys=p(Co,"Methods and tools for efficient training on a single GPU"),Co.forEach(t),Os=p(nl,` documentation page to learn about
performance optimization techniques.`),nl.forEach(t),this.h()},h(){f(T,"name","hf:doc:metadata"),f(T,"content",JSON.stringify(Ro)),f(I,"id","model-training-anatomy"),f(I,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(I,"href","#model-training-anatomy"),f(U,"class","relative group"),f(X,"href","https://huggingface.co/docs/datasets/v2.14.4/en/package_reference/main_classes#datasets.Dataset"),f(X,"rel","nofollow"),f(Ue,"href","/docs/transformers/v4.33.0/en/main_classes/trainer#transformers.Trainer"),f(R,"id","load-model"),f(R,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(R,"href","#load-model"),f(S,"class","relative group"),f(P,"id","memory-utilization-at-vanilla-training"),f(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(P,"href","#memory-utilization-at-vanilla-training"),f(g,"class","relative group"),f(Ce,"href","/docs/transformers/v4.33.0/en/main_classes/trainer#transformers.Trainer"),f(W,"id","anatomy-of-models-operations"),f(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(W,"href","#anatomy-of-models-operations"),f(E,"class","relative group"),f(de,"href","https://arxiv.org/abs/2007.00072"),f(de,"rel","nofollow"),f(B,"id","anatomy-of-models-memory"),f(B,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(B,"href","#anatomy-of-models-memory"),f(z,"class","relative group"),f(ye,"href","https://github.com/TimDettmers/bitsandbytes"),f(ye,"rel","nofollow"),f(De,"href","perf_train_gpu_one")},m(e,l){a(document.head,T),r(e,_,l),r(e,U,l),a(U,I),a(I,Oe),d(x,Oe,null),a(U,ol),a(U,He),a(He,rl),r(e,Vt,l),r(e,Te,l),a(Te,il),r(e,Zt,l),r(e,Je,l),a(Je,pl),r(e,Ft,l),d(D,e,l),r(e,xt,l),r(e,v,l),a(v,ml),a(v,Ke),a(Ke,Ml),a(v,fl),a(v,et),a(et,wl),a(v,hl),r(e,Dt,l),r(e,L,l),a(L,dl),a(L,X),a(X,cl),a(L,yl),r(e,Xt,l),d(q,e,l),r(e,qt,l),r(e,Q,l),a(Q,ul),a(Q,Ue),a(Ue,Tl),a(Q,Jl),r(e,Yt,l),d(Y,e,l),r(e,Ot,l),r(e,Ie,l),a(Ie,Ul),r(e,Ht,l),d(O,e,l),r(e,Kt,l),r(e,je,l),a(je,Il),r(e,ea,l),d(H,e,l),r(e,ta,l),r(e,be,l),a(be,jl),r(e,aa,l),r(e,S,l),a(S,R),a(R,tt),d(K,tt,null),a(S,bl),a(S,at),a(at,vl),r(e,la,l),r(e,G,l),a(G,Cl),a(G,lt),a(lt,Al),a(G,Sl),r(e,sa,l),d(ee,e,l),r(e,na,l),r(e,k,l),a(k,gl),a(k,st),a(st,El),a(k,zl),r(e,oa,l),d(te,e,l),r(e,ra,l),d(ae,e,l),r(e,ia,l),r(e,ve,l),a(ve,_l),r(e,pa,l),d(le,e,l),r(e,ma,l),d($,e,l),r(e,Ma,l),r(e,g,l),a(g,P),a(P,nt),d(se,nt,null),a(g,Ll),a(g,ot),a(ot,Ql),r(e,fa,l),r(e,N,l),a(N,Rl),a(N,Ce),a(Ce,Gl),a(N,kl),r(e,wa,l),d(ne,e,l),r(e,ha,l),d(oe,e,l),r(e,da,l),r(e,Ae,l),a(Ae,$l),r(e,ca,l),r(e,E,l),a(E,W),a(W,rt),d(re,rt,null),a(E,Pl),a(E,it),a(it,Nl),r(e,ya,l),r(e,Se,l),a(Se,Wl),r(e,ua,l),r(e,C,l),a(C,ie),a(ie,pt),a(pt,mt),a(mt,Bl),a(ie,Vl),a(ie,pe),a(pe,Zl),a(pe,Mt),a(Mt,Fl),a(pe,xl),a(C,Dl),a(C,me),a(me,ft),a(ft,wt),a(wt,Xl),a(me,ql),a(me,Me),a(Me,Yl),a(Me,ht),a(ht,Ol),a(Me,Hl),a(C,Kl),a(C,fe),a(fe,dt),a(dt,ct),a(ct,es),a(fe,ts),a(fe,we),a(we,as),a(we,yt),a(yt,ls),a(we,ss),r(e,Ta,l),r(e,ge,l),a(ge,ns),r(e,Ja,l),r(e,he,l),a(he,os),a(he,de),a(de,rs),r(e,Ua,l),r(e,z,l),a(z,B),a(B,ut),d(ce,ut,null),a(z,is),a(z,Tt),a(Tt,ps),r(e,Ia,l),r(e,Ee,l),a(Ee,ms),r(e,ja,l),r(e,J,l),a(J,Jt),a(Jt,Ms),a(J,fs),a(J,Ut),a(Ut,ws),a(J,hs),a(J,It),a(It,ds),a(J,cs),a(J,jt),a(jt,ys),a(J,us),a(J,bt),a(bt,Ts),a(J,Js),a(J,vt),a(vt,Us),r(e,ba,l),r(e,ze,l),a(ze,Is),r(e,va,l),r(e,_e,l),a(_e,js),r(e,Ca,l),r(e,Le,l),a(Le,Ct),a(Ct,bs),r(e,Aa,l),r(e,V,l),a(V,At),a(At,vs),a(V,Cs),a(V,St),a(St,As),r(e,Sa,l),r(e,Qe,l),a(Qe,gt),a(gt,Ss),r(e,ga,l),r(e,A,l),a(A,Et),a(Et,gs),a(A,Es),a(A,Re),a(Re,zs),a(Re,ye),a(ye,_s),a(A,Ls),a(A,zt),a(zt,Qs),r(e,Ea,l),r(e,Ge,l),a(Ge,_t),a(_t,Rs),r(e,za,l),r(e,ke,l),a(ke,Lt),a(Lt,Gs),r(e,_a,l),r(e,$e,l),a($e,Qt),a(Qt,ks),r(e,La,l),r(e,Pe,l),a(Pe,Rt),a(Rt,$s),r(e,Qa,l),r(e,Ne,l),a(Ne,Ps),r(e,Ra,l),r(e,We,l),a(We,Gt),a(Gt,Ns),r(e,Ga,l),r(e,Be,l),a(Be,Ws),r(e,ka,l),r(e,Ve,l),a(Ve,kt),a(kt,Bs),r(e,$a,l),r(e,Ze,l),a(Ze,Vs),r(e,Pa,l),r(e,Fe,l),a(Fe,Z),a(Z,$t),a($t,Zs),a(Z,Fs),a(Z,Pt),a(Pt,xs),a(Z,Ds),r(e,Na,l),r(e,xe,l),a(xe,Xs),r(e,Wa,l),r(e,F,l),a(F,qs),a(F,De),a(De,Ys),a(F,Os),Ba=!0},p(e,[l]){const ue={};l&2&&(ue.$$scope={dirty:l,ctx:e}),$.$set(ue)},i(e){Ba||(c(x.$$.fragment,e),c(D.$$.fragment,e),c(q.$$.fragment,e),c(Y.$$.fragment,e),c(O.$$.fragment,e),c(H.$$.fragment,e),c(K.$$.fragment,e),c(ee.$$.fragment,e),c(te.$$.fragment,e),c(ae.$$.fragment,e),c(le.$$.fragment,e),c($.$$.fragment,e),c(se.$$.fragment,e),c(ne.$$.fragment,e),c(oe.$$.fragment,e),c(re.$$.fragment,e),c(ce.$$.fragment,e),Ba=!0)},o(e){y(x.$$.fragment,e),y(D.$$.fragment,e),y(q.$$.fragment,e),y(Y.$$.fragment,e),y(O.$$.fragment,e),y(H.$$.fragment,e),y(K.$$.fragment,e),y(ee.$$.fragment,e),y(te.$$.fragment,e),y(ae.$$.fragment,e),y(le.$$.fragment,e),y($.$$.fragment,e),y(se.$$.fragment,e),y(ne.$$.fragment,e),y(oe.$$.fragment,e),y(re.$$.fragment,e),y(ce.$$.fragment,e),Ba=!1},d(e){t(T),e&&t(_),e&&t(U),u(x),e&&t(Vt),e&&t(Te),e&&t(Zt),e&&t(Je),e&&t(Ft),u(D,e),e&&t(xt),e&&t(v),e&&t(Dt),e&&t(L),e&&t(Xt),u(q,e),e&&t(qt),e&&t(Q),e&&t(Yt),u(Y,e),e&&t(Ot),e&&t(Ie),e&&t(Ht),u(O,e),e&&t(Kt),e&&t(je),e&&t(ea),u(H,e),e&&t(ta),e&&t(be),e&&t(aa),e&&t(S),u(K),e&&t(la),e&&t(G),e&&t(sa),u(ee,e),e&&t(na),e&&t(k),e&&t(oa),u(te,e),e&&t(ra),u(ae,e),e&&t(ia),e&&t(ve),e&&t(pa),u(le,e),e&&t(ma),u($,e),e&&t(Ma),e&&t(g),u(se),e&&t(fa),e&&t(N),e&&t(wa),u(ne,e),e&&t(ha),u(oe,e),e&&t(da),e&&t(Ae),e&&t(ca),e&&t(E),u(re),e&&t(ya),e&&t(Se),e&&t(ua),e&&t(C),e&&t(Ta),e&&t(ge),e&&t(Ja),e&&t(he),e&&t(Ua),e&&t(z),u(ce),e&&t(Ia),e&&t(Ee),e&&t(ja),e&&t(J),e&&t(ba),e&&t(ze),e&&t(va),e&&t(_e),e&&t(Ca),e&&t(Le),e&&t(Aa),e&&t(V),e&&t(Sa),e&&t(Qe),e&&t(ga),e&&t(A),e&&t(Ea),e&&t(Ge),e&&t(za),e&&t(ke),e&&t(_a),e&&t($e),e&&t(La),e&&t(Pe),e&&t(Qa),e&&t(Ne),e&&t(Ra),e&&t(We),e&&t(Ga),e&&t(Be),e&&t(ka),e&&t(Ve),e&&t($a),e&&t(Ze),e&&t(Pa),e&&t(Fe),e&&t(Na),e&&t(xe),e&&t(Wa),e&&t(F)}}}const Ro={local:"model-training-anatomy",sections:[{local:"load-model",title:"Load Model"},{local:"memory-utilization-at-vanilla-training",title:"Memory utilization at vanilla training"},{local:"anatomy-of-models-operations",title:"Anatomy of Model's Operations"},{local:"anatomy-of-models-memory",title:"Anatomy of Model's Memory"}],title:"Model training anatomy"};function Go(Bt){return zo(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wo extends Ao{constructor(T){super();So(this,T,Go,Qo,go,{})}}export{Wo as default,Ro as metadata};
