import{S as V3,i as H3,s as O3,e as l,k as p,w as c,t as s,M as q3,c as r,d as t,m as f,a as o,x as d,h as i,b as m,N as S2,G as a,g as h,y as u,q as w,o as y,B as b,v as K3}from"../chunks/vendor-hf-doc-builder.js";import{T as X3}from"../chunks/Tip-hf-doc-builder.js";import{I as v}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as M}from"../chunks/CodeBlock-hf-doc-builder.js";function eA(zr){let g,E;return{c(){g=l("p"),E=s("Note: In order to properly clear the memory after experiments we need restart the Python kernel between experiments. Run all steps above and then just one of the experiments below.")},l(U){g=r(U,"P",{});var $=o(g);E=i($,"Note: In order to properly clear the memory after experiments we need restart the Python kernel between experiments. Run all steps above and then just one of the experiments below."),$.forEach(t)},m(U,$){h(U,g,$),a(g,E)},d(U){U&&t(g)}}}function tA(zr){let g;return{c(){g=s(`Note that in order to use the 8-bit optimizer with an existing pretrained model a change to the embedding layer is needed.
Read [this issue](https://github.com/huggingface/transformers/issues/14819) for more information.`)},l(E){g=i(E,`Note that in order to use the 8-bit optimizer with an existing pretrained model a change to the embedding layer is needed.
Read [this issue](https://github.com/huggingface/transformers/issues/14819) for more information.`)},m(E,U){h(E,g,U)},d(E){E&&t(g)}}}function aA(zr){let g,E,U,$,Mi,Ot,uu,vi,wu,rp,Be,yu,Cr,bu,Mu,op,N,vu,Gr,gu,Tu,qt,Uu,_u,sp,Ne,gi,le,Sr,Ju,Eu,Rr,Au,ju,Pr,Iu,ku,_,re,Wr,$u,zu,Br,Cu,Gu,Nr,Su,Ru,oe,Qr,Pu,Wu,Zr,Bu,Nu,Fr,Qu,Zu,se,xr,Fu,xu,Yr,Yu,Lu,Lr,Du,Xu,ie,Dr,Vu,Hu,Xr,Ou,qu,Vr,Ku,ew,ne,Hr,tw,aw,Or,lw,rw,qr,ow,sw,he,Kr,iw,nw,eo,hw,pw,to,fw,mw,pe,ao,cw,dw,lo,uw,ww,ro,yw,ip,oo,bw,np,Kt,hp,Q,Mw,Ti,vw,gw,Ui,Tw,Uw,pp,Qe,_w,ea,Jw,Ew,fp,ta,mp,Ze,Aw,so,jw,Iw,cp,aa,dp,io,kw,up,la,wp,no,$w,yp,ra,bp,ho,zw,Mp,fe,Fe,_i,oa,Cw,Ji,Gw,vp,xe,Sw,Ei,Rw,Pw,gp,sa,Tp,Ye,Ww,Ai,Bw,Nw,Up,ia,_p,na,Jp,po,Qw,Ep,ha,Ap,Le,jp,me,De,ji,pa,Zw,Ii,Fw,Ip,Xe,xw,fo,Yw,Lw,kp,fa,$p,ma,zp,mo,Dw,Cp,ce,Ve,ki,ca,Xw,$i,Vw,Gp,co,Hw,Sp,Z,da,zi,Ci,Ow,qw,ua,Kw,Gi,ey,ty,ay,wa,Si,Ri,ly,ry,ya,oy,Pi,sy,iy,ny,ba,Wi,Bi,hy,py,Ma,fy,Ni,my,cy,Rp,uo,dy,Pp,va,uy,ga,wy,Wp,de,He,Qi,Ta,yy,Zi,by,Bp,wo,My,Np,yo,vy,Qp,bo,Fi,gy,Zp,Oe,xi,Ty,Uy,Yi,_y,Fp,Mo,Li,Jy,xp,F,Di,Ey,Ay,vo,jy,Ua,Iy,ky,Xi,$y,Yp,go,Vi,zy,Lp,To,Hi,Cy,Dp,Uo,Oi,Gy,Xp,_o,qi,Sy,Vp,Jo,Ry,Hp,Eo,Ki,Py,Op,Ao,Wy,qp,jo,en,By,Kp,Io,Ny,ef,ko,qe,tn,Qy,Zy,an,Fy,xy,tf,$o,Yy,af,zo,Ly,lf,ue,Ke,ln,_a,Dy,rn,Xy,rf,Co,Vy,of,x,Hy,Ja,Oy,qy,Ea,Ky,eb,sf,Aa,ja,tb,ab,nf,et,lb,Ia,rb,ob,hf,we,tt,on,ka,sb,sn,ib,pf,Go,nb,ff,z,hb,So,pb,fb,nn,mb,cb,Ro,db,ub,mf,$a,cf,za,df,A,wb,hn,yb,bb,pn,Mb,vb,fn,gb,Tb,mn,Ub,_b,uf,Y,Jb,Ca,Eb,Ab,Ga,jb,Ib,wf,Po,kb,yf,ye,at,cn,Sa,$b,dn,zb,bf,Wo,Cb,Mf,lt,Gb,Ra,Sb,Rb,vf,L,Pb,Bo,Wb,Bb,No,Nb,Qb,gf,Pa,Tf,Wa,Uf,Qo,Zb,_f,be,rt,un,Ba,Fb,wn,xb,Jf,Zo,Yb,Ef,C,Na,Lb,yn,Db,Xb,Vb,Qa,Hb,bn,Ob,qb,Kb,Za,eM,Mn,tM,aM,lM,vn,rM,Af,Fo,oM,jf,Me,xo,R2,sM,Fa,iM,nM,If,Yo,hM,kf,ve,ot,gn,xa,pM,Tn,fM,$f,D,mM,Un,cM,dM,_n,uM,wM,zf,Ya,Cf,La,Gf,Lo,yM,Sf,Da,Rf,Xa,Pf,Do,bM,Wf,ge,st,Jn,Va,MM,En,vM,Bf,Xo,gM,Nf,Ha,Qf,Te,it,An,Oa,TM,jn,UM,Zf,Vo,_M,Ff,qa,xf,Ho,JM,Yf,nt,EM,Ka,AM,jM,Lf,Oo,IM,Df,qo,kM,Xf,el,Vf,Ko,$M,Hf,X,zM,In,CM,GM,kn,SM,RM,Of,ht,PM,$n,WM,BM,qf,V,NM,tl,QM,ZM,al,FM,xM,Kf,es,YM,em,Ue,pt,zn,ll,LM,Cn,DM,tm,ts,XM,am,ft,VM,Gn,HM,OM,lm,as,qM,rm,rl,om,H,KM,ol,ev,tv,Sn,av,lv,sm,mt,rv,sl,ov,sv,im,ct,iv,Rn,nv,hv,nm,O,il,pv,Pn,fv,mv,cv,nl,dv,Wn,uv,wv,yv,hl,bv,Bn,Mv,vv,hm,ls,gv,pm,_e,dt,Nn,pl,Tv,Qn,Uv,fm,ut,_v,Zn,Jv,Ev,mm,fl,cm,ml,dm,rs,Av,um,cl,wm,dl,ym,os,jv,bm,Je,wt,Fn,ul,Iv,xn,kv,Mm,ss,$v,vm,j,zv,is,Cv,Gv,ns,Sv,Rv,wl,Pv,Wv,Yn,Bv,Nv,gm,hs,Qv,Tm,yt,Um,yl,_m,bt,Zv,Ln,Fv,xv,Jm,bl,Em,Ml,Am,ps,Yv,jm,vl,Im,gl,km,fs,Lv,$m,ms,cs,P2,zm,Ee,Mt,Dn,Tl,Dv,Xn,Vn,Xv,Cm,Ae,vt,Hn,Ul,Vv,On,Hv,Gm,q,Ov,ds,qv,Kv,us,eg,tg,Sm,_l,Rm,ws,ag,Pm,Jl,Wm,J,lg,El,qn,rg,og,ys,sg,ig,Al,Kn,ng,hg,eh,pg,fg,jl,th,mg,cg,Bm,gt,dg,ah,ug,wg,Nm,Il,Qm,Tt,yg,kl,bg,Mg,Zm,je,Ut,lh,$l,vg,rh,gg,Fm,bs,Tg,xm,_t,Ms,oh,Ug,_g,Jg,vs,sh,Eg,Ag,Ym,Ie,Jt,ih,zl,jg,nh,Ig,Lm,Et,kg,gs,$g,zg,Dm,Ts,Cg,Xm,At,hh,Gg,Sg,ph,Rg,Vm,jt,Pg,Us,Wg,Bg,Hm,_s,Ng,Om,G,fh,Cl,Qg,Js,Zg,Fg,xg,Gl,mh,Yg,Lg,Sl,Dg,ch,Rl,Xg,Es,Vg,Hg,Og,dh,ke,qg,Pl,Kg,e1,uh,t1,a1,qm,$e,It,wh,Wl,l1,yh,r1,Km,As,o1,ec,ze,kt,bh,Bl,s1,Mh,i1,tc,js,n1,ac,$t,vh,h1,p1,gh,f1,lc,Is,m1,rc,ks,c1,oc,Ce,zt,Th,Nl,d1,Uh,u1,sc,Ct,w1,Ql,y1,b1,ic,Gt,M1,_h,v1,g1,nc,$s,T1,hc,K,U1,Zl,_1,J1,Fl,E1,A1,pc,zs,j1,fc,Ge,St,Jh,xl,I1,Eh,k1,mc,Se,Rt,Ah,Yl,$1,jh,z1,cc,Cs,C1,dc,Gs,G1,uc,Ss,S1,wc,Rs,Ps,W2,yc,Pt,R1,Ll,P1,W1,bc,Ws,B1,Mc,Bs,N1,vc,Ns,Q1,gc,Qs,Z1,Tc,ee,Ih,Dl,F1,x1,kh,Xl,Y1,L1,$h,Vl,D1,Uc,T,X1,Hl,V1,H1,Ol,O1,q1,ql,K1,eT,Kl,tT,aT,er,lT,rT,Zs,oT,sT,_c,Re,Wt,zh,tr,iT,Ch,nT,Jc,Fs,hT,Ec,Bt,pT,xs,fT,mT,Ac,Pe,Nt,Gh,ar,cT,Sh,dT,jc,S,uT,lr,Rh,wT,yT,rr,bT,MT,or,vT,gT,Ic,Qt,TT,sr,Ph,UT,_T,kc,ir,$c,Ys,JT,zc,We,Zt,Wh,nr,ET,Bh,AT,Cc,Ft,jT,hr,IT,kT,Gc,B,Nh,$T,zT,pr,CT,GT,Qh,ST,RT,Sc,Ls,PT,Rc,fr,Zh,WT,BT,Pc,xt,Ds,Fh,NT,QT,ZT,Xs,xh,FT,xT,Wc,mr,Yh,YT,LT,Bc,R,cr,Lh,DT,XT,dr,VT,HT,ur,Dh,OT,qT,wr,KT,e2,yr,Xh,t2,a2,br,l2,r2,Mr,Vh,o2,s2,vr,i2,Nc,gr,Hh,n2,h2,Qc,P,Tr,Oh,p2,f2,Ur,m2,c2,_r,qh,d2,u2,Jr,w2,y2,Er,Kh,b2,M2,Ar,v2,g2,jr,ep,T2,U2,Ir,_2,Zc;return Ot=new v({}),Kt=new M({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwYWNjZWxlcmF0ZSUyMG52aWRpYS1tbC1weTM=",highlighted:"pip install transformers datasets accelerate nvidia-ml-py3"}}),ta=new M({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQSUwQSUwQXNlcV9sZW4lMkMlMjBkYXRhc2V0X3NpemUlMjAlM0QlMjA1MTIlMkMlMjA1MTIlMEFkdW1teV9kYXRhJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIyaW5wdXRfaWRzJTIyJTNBJTIwbnAucmFuZG9tLnJhbmRpbnQoMTAwJTJDJTIwMzAwMDAlMkMlMjAoZGF0YXNldF9zaXplJTJDJTIwc2VxX2xlbikpJTJDJTBBJTIwJTIwJTIwJTIwJTIybGFiZWxzJTIyJTNBJTIwbnAucmFuZG9tLnJhbmRpbnQoMCUyQyUyMDElMkMlMjAoZGF0YXNldF9zaXplKSklMkMlMEElN0QlMEFkcyUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KGR1bW15X2RhdGEpJTBBZHMuc2V0X2Zvcm1hdCglMjJwdCUyMik=",highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset


seq_len, dataset_size = <span class="hljs-number">512</span>, <span class="hljs-number">512</span>
dummy_data = {
    <span class="hljs-string">&quot;input_ids&quot;</span>: np.random.randint(<span class="hljs-number">100</span>, <span class="hljs-number">30000</span>, (dataset_size, seq_len)),
    <span class="hljs-string">&quot;labels&quot;</span>: np.random.randint(<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, (dataset_size)),
}
ds = Dataset.from_dict(dummy_data)
ds.set_format(<span class="hljs-string">&quot;pt&quot;</span>)`}}),aa=new M({props:{code:"ZnJvbSUyMHB5bnZtbCUyMGltcG9ydCUyMColMEElMEElMEFkZWYlMjBwcmludF9ncHVfdXRpbGl6YXRpb24oKSUzQSUwQSUyMCUyMCUyMCUyMG52bWxJbml0KCklMEElMjAlMjAlMjAlMjBoYW5kbGUlMjAlM0QlMjBudm1sRGV2aWNlR2V0SGFuZGxlQnlJbmRleCgwKSUwQSUyMCUyMCUyMCUyMGluZm8lMjAlM0QlMjBudm1sRGV2aWNlR2V0TWVtb3J5SW5mbyhoYW5kbGUpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMkdQVSUyMG1lbW9yeSUyMG9jY3VwaWVkJTNBJTIwJTdCaW5mby51c2VkJTJGJTJGMTAyNCoqMiU3RCUyME1CLiUyMiklMEElMEElMEFkZWYlMjBwcmludF9zdW1tYXJ5KHJlc3VsdCklM0ElMEElMjAlMjAlMjAlMjBwcmludChmJTIyVGltZSUzQSUyMCU3QnJlc3VsdC5tZXRyaWNzJTVCJ3RyYWluX3J1bnRpbWUnJTVEJTNBLjJmJTdEJTIyKSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjJTYW1wbGVzJTJGc2Vjb25kJTNBJTIwJTdCcmVzdWx0Lm1ldHJpY3MlNUIndHJhaW5fc2FtcGxlc19wZXJfc2Vjb25kJyU1RCUzQS4yZiU3RCUyMiklMEElMjAlMjAlMjAlMjBwcmludF9ncHVfdXRpbGl6YXRpb24oKQ==",highlighted:`<span class="hljs-keyword">from</span> pynvml <span class="hljs-keyword">import</span> *


<span class="hljs-keyword">def</span> <span class="hljs-title function_">print_gpu_utilization</span>():
    nvmlInit()
    handle = nvmlDeviceGetHandleByIndex(<span class="hljs-number">0</span>)
    info = nvmlDeviceGetMemoryInfo(handle)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;GPU memory occupied: <span class="hljs-subst">{info.used//<span class="hljs-number">1024</span>**<span class="hljs-number">2</span>}</span> MB.&quot;</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">print_summary</span>(<span class="hljs-params">result</span>):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Time: <span class="hljs-subst">{result.metrics[<span class="hljs-string">&#x27;train_runtime&#x27;</span>]:<span class="hljs-number">.2</span>f}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Samples/second: <span class="hljs-subst">{result.metrics[<span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>]:<span class="hljs-number">.2</span>f}</span>&quot;</span>)
    print_gpu_utilization()`}}),la=new M({props:{code:"cHJpbnRfZ3B1X3V0aWxpemF0aW9uKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>print_gpu_utilization()
GPU memory occupied: <span class="hljs-number">0</span> MB.`}}),ra=new M({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEElMEF0b3JjaC5vbmVzKCgxJTJDJTIwMSkpLnRvKCUyMmN1ZGElMjIpJTBBcHJpbnRfZ3B1X3V0aWxpemF0aW9uKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch


<span class="hljs-meta">&gt;&gt;&gt; </span>torch.ones((<span class="hljs-number">1</span>, <span class="hljs-number">1</span>)).to(<span class="hljs-string">&quot;cuda&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>print_gpu_utilization()
GPU memory occupied: <span class="hljs-number">1343</span> MB.`}}),oa=new v({}),sa=new M({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmJlcnQtbGFyZ2UtdW5jYXNlZCUyMikudG8oJTIyY3VkYSUyMiklMEFwcmludF9ncHVfdXRpbGl6YXRpb24oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification


<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-large-uncased&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>print_gpu_utilization()
GPU memory occupied: <span class="hljs-number">2631</span> MB.`}}),ia=new M({props:{code:"bnZpZGlhLXNtaQ==",highlighted:"nvidia-smi"}}),na=new M({props:{code:"VHVlJTIwSmFuJTIwMTElMjAwOCUzQTU4JTNBMDUlMjAyMDIyJTBBJTJCLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0lMkIlMEElN0MlMjBOVklESUEtU01JJTIwNDYwLjkxLjAzJTIwJTIwJTIwJTIwRHJpdmVyJTIwVmVyc2lvbiUzQSUyMDQ2MC45MS4wMyUyMCUyMCUyMCUyMENVREElMjBWZXJzaW9uJTNBJTIwMTEuMiUyMCUyMCUyMCUyMCUyMCU3QyUwQSU3Qy0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0lMkItLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTJCLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSUyQiUwQSU3QyUyMEdQVSUyMCUyME5hbWUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBQZXJzaXN0ZW5jZS1NJTdDJTIwQnVzLUlkJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwRGlzcC5BJTIwJTdDJTIwVm9sYXRpbGUlMjBVbmNvcnIuJTIwRUNDJTIwJTdDJTBBJTdDJTIwRmFuJTIwJTIwVGVtcCUyMCUyMFBlcmYlMjAlMjBQd3IlM0FVc2FnZSUyRkNhcCU3QyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME1lbW9yeS1Vc2FnZSUyMCU3QyUyMEdQVS1VdGlsJTIwJTIwQ29tcHV0ZSUyME0uJTIwJTdDJTBBJTdDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTUlHJTIwTS4lMjAlN0MlMEElN0MlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMkIlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMkIlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlN0MlMEElN0MlMjAlMjAlMjAwJTIwJTIwVGVzbGElMjBWMTAwLVNYTTIuLi4lMjAlMjBPbiUyMCUyMCUyMCU3QyUyMDAwMDAwMDAwJTNBMDAlM0EwNC4wJTIwT2ZmJTIwJTdDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwMCUyMCU3QyUwQSU3QyUyME4lMkZBJTIwJTIwJTIwMzdDJTIwJTIwJTIwJTIwUDAlMjAlMjAlMjAlMjAzOVclMjAlMkYlMjAzMDBXJTIwJTdDJTIwJTIwJTIwMjYzMU1pQiUyMCUyRiUyMDE2MTYwTWlCJTIwJTdDJTIwJTIwJTIwJTIwJTIwJTIwMCUyNSUyMCUyMCUyMCUyMCUyMCUyMERlZmF1bHQlMjAlN0MlMEElN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBOJTJGQSUyMCU3QyUwQSUyQi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0lMkItLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTJCLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSUyQiUwQSUwQSUyQi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTJCJTBBJTdDJTIwUHJvY2Vzc2VzJTNBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdDJTBBJTdDJTIwJTIwR1BVJTIwJTIwJTIwR0klMjAlMjAlMjBDSSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFBJRCUyMCUyMCUyMFR5cGUlMjAlMjAlMjBQcm9jZXNzJTIwbmFtZSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEdQVSUyME1lbW9yeSUyMCU3QyUwQSU3QyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMElEJTIwJTIwJTIwSUQlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBVc2FnZSUyMCUyMCUyMCUyMCUyMCUyMCU3QyUwQSU3QyUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCU3QyUwQSU3QyUyMCUyMCUyMCUyMDAlMjAlMjAlMjBOJTJGQSUyMCUyME4lMkZBJTIwJTIwJTIwJTIwJTIwJTIwMzcyMSUyMCUyMCUyMCUyMCUyMCUyMEMlMjAlMjAlMjAuLi5udnMlMkZjb2RlcGFycm90JTJGYmluJTJGcHl0aG9uJTIwJTIwJTIwJTIwJTIwMjYyOU1pQiUyMCU3QyUwQSUyQi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTJC",highlighted:`Tue Jan 11 08:58:05 2022
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 460.91.03    Driver Version: 460.91.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla V100-SXM2...  On   | 00000000:00:04.0 Off |                    0 |
| N/A   37C    P0    39W / 300W |   2631MiB / 16160MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A      3721      C   ...nvs/codeparrot/bin/python     2629MiB |
+-----------------------------------------------------------------------------+`}}),ha=new M({props:{code:"ZGVmYXVsdF9hcmdzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIyb3V0cHV0X2RpciUyMiUzQSUyMCUyMnRtcCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmV2YWx1YXRpb25fc3RyYXRlZ3klMjIlM0ElMjAlMjJzdGVwcyUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMm51bV90cmFpbl9lcG9jaHMlMjIlM0ElMjAxJTJDJTBBJTIwJTIwJTIwJTIwJTIybG9nX2xldmVsJTIyJTNBJTIwJTIyZXJyb3IlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJyZXBvcnRfdG8lMjIlM0ElMjAlMjJub25lJTIyJTJDJTBBJTdE",highlighted:`default_args = {
    <span class="hljs-string">&quot;output_dir&quot;</span>: <span class="hljs-string">&quot;tmp&quot;</span>,
    <span class="hljs-string">&quot;evaluation_strategy&quot;</span>: <span class="hljs-string">&quot;steps&quot;</span>,
    <span class="hljs-string">&quot;num_train_epochs&quot;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&quot;log_level&quot;</span>: <span class="hljs-string">&quot;error&quot;</span>,
    <span class="hljs-string">&quot;report_to&quot;</span>: <span class="hljs-string">&quot;none&quot;</span>,
}`}}),Le=new X3({props:{$$slots:{default:[eA]},$$scope:{ctx:zr}}}),pa=new v({}),fa=new M({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTJDJTIwVHJhaW5lciUyQyUyMGxvZ2dpbmclMEElMEFsb2dnaW5nLnNldF92ZXJib3NpdHlfZXJyb3IoKSUwQSUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyhwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0Q0JTJDJTIwKipkZWZhdWx0X2FyZ3MpJTBBdHJhaW5lciUyMCUzRCUyMFRyYWluZXIobW9kZWwlM0Rtb2RlbCUyQyUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTIwdHJhaW5fZGF0YXNldCUzRGRzKSUwQXJlc3VsdCUyMCUzRCUyMHRyYWluZXIudHJhaW4oKSUwQXByaW50X3N1bW1hcnkocmVzdWx0KQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, Trainer, logging

logging.set_verbosity_error()


training_args = TrainingArguments(per_device_train_batch_size=<span class="hljs-number">4</span>, **default_args)
trainer = Trainer(model=model, args=training_args, train_dataset=ds)
result = trainer.train()
print_summary(result)`}}),ma=new M({props:{code:"VGltZSUzQSUyMDU3LjgyJTBBU2FtcGxlcyUyRnNlY29uZCUzQSUyMDguODYlMEFHUFUlMjBtZW1vcnklMjBvY2N1cGllZCUzQSUyMDE0OTQ5JTIwTUIu",highlighted:`<span class="hljs-keyword">Time:</span> 57.82
Samples/second: 8.86
GPU memory occupied: 14949 MB.`}}),ca=new v({}),Ta=new v({}),_a=new v({}),ka=new v({}),$a=new M({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMkMlMjBncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0Q0JTJDJTIwKipkZWZhdWx0X2FyZ3MpJTBBJTBBdHJhaW5lciUyMCUzRCUyMFRyYWluZXIobW9kZWwlM0Rtb2RlbCUyQyUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTIwdHJhaW5fZGF0YXNldCUzRGRzKSUwQXJlc3VsdCUyMCUzRCUyMHRyYWluZXIudHJhaW4oKSUwQXByaW50X3N1bW1hcnkocmVzdWx0KQ==",highlighted:`training_args = TrainingArguments(per_device_train_batch_size=<span class="hljs-number">1</span>, gradient_accumulation_steps=<span class="hljs-number">4</span>, **default_args)

trainer = Trainer(model=model, args=training_args, train_dataset=ds)
result = trainer.train()
print_summary(result)`}}),za=new M({props:{code:"VGltZSUzQSUyMDY2LjAzJTBBU2FtcGxlcyUyRnNlY29uZCUzQSUyMDcuNzUlMEFHUFUlMjBtZW1vcnklMjBvY2N1cGllZCUzQSUyMDg2ODElMjBNQi4=",highlighted:`<span class="hljs-keyword">Time:</span> 66.03
Samples/second: 7.75
GPU memory occupied: 8681 MB.`}}),Sa=new v({}),Pa=new M({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMkMlMjBncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0Q0JTJDJTIwZ3JhZGllbnRfY2hlY2twb2ludGluZyUzRFRydWUlMkMlMjAqKmRlZmF1bHRfYXJncyUwQSklMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lcihtb2RlbCUzRG1vZGVsJTJDJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMjB0cmFpbl9kYXRhc2V0JTNEZHMpJTBBcmVzdWx0JTIwJTNEJTIwdHJhaW5lci50cmFpbigpJTBBcHJpbnRfc3VtbWFyeShyZXN1bHQp",highlighted:`training_args = TrainingArguments(
    per_device_train_batch_size=<span class="hljs-number">1</span>, gradient_accumulation_steps=<span class="hljs-number">4</span>, gradient_checkpointing=<span class="hljs-literal">True</span>, **default_args
)

trainer = Trainer(model=model, args=training_args, train_dataset=ds)
result = trainer.train()
print_summary(result)`}}),Wa=new M({props:{code:"VGltZSUzQSUyMDg1LjQ3JTBBU2FtcGxlcyUyRnNlY29uZCUzQSUyMDUuOTklMEFHUFUlMjBtZW1vcnklMjBvY2N1cGllZCUzQSUyMDY3NzUlMjBNQi4=",highlighted:`<span class="hljs-keyword">Time:</span> 85.47
Samples/second: 5.99
GPU memory occupied: 6775 MB.`}}),Ba=new v({}),xa=new v({}),Ya=new M({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDQlMkMlMjBmcDE2JTNEVHJ1ZSUyQyUyMCoqZGVmYXVsdF9hcmdzKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKG1vZGVsJTNEbW9kZWwlMkMlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUyMHRyYWluX2RhdGFzZXQlM0RkcyklMEFyZXN1bHQlMjAlM0QlMjB0cmFpbmVyLnRyYWluKCklMEFwcmludF9zdW1tYXJ5KHJlc3VsdCk=",highlighted:`training_args = TrainingArguments(per_device_train_batch_size=<span class="hljs-number">4</span>, fp16=<span class="hljs-literal">True</span>, **default_args)

trainer = Trainer(model=model, args=training_args, train_dataset=ds)
result = trainer.train()
print_summary(result)`}}),La=new M({props:{code:"VGltZSUzQSUyMDI3LjQ2JTBBU2FtcGxlcyUyRnNlY29uZCUzQSUyMDE4LjY0JTBBR1BVJTIwbWVtb3J5JTIwb2NjdXBpZWQlM0ElMjAxMzkzOSUyME1CLg==",highlighted:`<span class="hljs-keyword">Time:</span> 27.46
Samples/second: 18.64
GPU memory occupied: 13939 MB.`}}),Da=new M({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMkMlMEElMjAlMjAlMjAlMjBncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfY2hlY2twb2ludGluZyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBmcDE2JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCoqZGVmYXVsdF9hcmdzJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKG1vZGVsJTNEbW9kZWwlMkMlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUyMHRyYWluX2RhdGFzZXQlM0RkcyklMEFyZXN1bHQlMjAlM0QlMjB0cmFpbmVyLnRyYWluKCklMEFwcmludF9zdW1tYXJ5KHJlc3VsdCk=",highlighted:`training_args = TrainingArguments(
    per_device_train_batch_size=<span class="hljs-number">1</span>,
    gradient_accumulation_steps=<span class="hljs-number">4</span>,
    gradient_checkpointing=<span class="hljs-literal">True</span>,
    fp16=<span class="hljs-literal">True</span>,
    **default_args,
)

trainer = Trainer(model=model, args=training_args, train_dataset=ds)
result = trainer.train()
print_summary(result)`}}),Xa=new M({props:{code:"VGltZSUzQSUyMDUwLjc2JTBBU2FtcGxlcyUyRnNlY29uZCUzQSUyMDEwLjA5JTBBR1BVJTIwbWVtb3J5JTIwb2NjdXBpZWQlM0ElMjA3Mjc1JTIwTUIu",highlighted:`<span class="hljs-keyword">Time:</span> 50.76
Samples/second: 10.09
GPU memory occupied: 7275 MB.`}}),Va=new v({}),Ha=new M({props:{code:"VHJhaW5pbmdBcmd1bWVudHMoYmYxNiUzRFRydWUp",highlighted:'TrainingArguments(bf16=<span class="hljs-literal">True</span>)'}}),Oa=new v({}),qa=new M({props:{code:"aW1wb3J0JTIwdG9yY2glMEF0b3JjaC5iYWNrZW5kcy5jdWRhLm1hdG11bC5hbGxvd190ZjMyJTIwJTNEJTIwVHJ1ZQ==",highlighted:`import torch
torch<span class="hljs-selector-class">.backends</span><span class="hljs-selector-class">.cuda</span><span class="hljs-selector-class">.matmul</span><span class="hljs-selector-class">.allow_tf32</span> = True`}}),el=new M({props:{code:"VHJhaW5pbmdBcmd1bWVudHModGYzMiUzRFRydWUp",highlighted:'TrainingArguments(tf32=<span class="hljs-literal">True</span>)'}}),ll=new v({}),rl=new M({props:{code:"JTI0JTIwcHl0aG9uJTIwZXhhbXBsZXMlMkZweXRvcmNoJTJGdHJhbnNsYXRpb24lMkZydW5fdHJhbnNsYXRpb24ucHklMjAtaCUyMCU3QyUyMGdyZXAlMjAlMjIlNUMtb3B0aW0lMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUItLW9wdGltJTIwJTdCYWRhbXdfaGYlMkNhZGFtd190b3JjaCUyQ2FkYW13X3RvcmNoX3hsYSUyQ2FkYW13X2FwZXhfZnVzZWQlMkNhZGFmYWN0b3IlN0QlNUQ=",highlighted:`$ python examples/pytorch/translation/run_translation.py -h | grep <span class="hljs-string">&quot;\\-optim&quot;</span>
         [--optim {adamw_hf,adamw_torch,adamw_torch_xla,adamw_apex_fused,adafactor}]`}}),pl=new v({}),fl=new M({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDQlMkMlMjBvcHRpbSUzRCUyMmFkYWZhY3RvciUyMiUyQyUyMCoqZGVmYXVsdF9hcmdzKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKG1vZGVsJTNEbW9kZWwlMkMlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUyMHRyYWluX2RhdGFzZXQlM0RkcyklMEFyZXN1bHQlMjAlM0QlMjB0cmFpbmVyLnRyYWluKCklMEFwcmludF9zdW1tYXJ5KHJlc3VsdCk=",highlighted:`training_args = TrainingArguments(per_device_train_batch_size=<span class="hljs-number">4</span>, optim=<span class="hljs-string">&quot;adafactor&quot;</span>, **default_args)

trainer = Trainer(model=model, args=training_args, train_dataset=ds)
result = trainer.train()
print_summary(result)`}}),ml=new M({props:{code:"VGltZSUzQSUyMDY0LjMxJTBBU2FtcGxlcyUyRnNlY29uZCUzQSUyMDcuOTYlMEFHUFUlMjBtZW1vcnklMjBvY2N1cGllZCUzQSUyMDEyMjk1JTIwTUIu",highlighted:`<span class="hljs-keyword">Time:</span> 64.31
Samples/second: 7.96
GPU memory occupied: 12295 MB.`}}),cl=new M({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMkMlMEElMjAlMjAlMjAlMjBncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfY2hlY2twb2ludGluZyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBmcDE2JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMG9wdGltJTNEJTIyYWRhZmFjdG9yJTIyJTJDJTBBJTIwJTIwJTIwJTIwKipkZWZhdWx0X2FyZ3MlMkMlMEEpJTBBJTBBdHJhaW5lciUyMCUzRCUyMFRyYWluZXIobW9kZWwlM0Rtb2RlbCUyQyUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTIwdHJhaW5fZGF0YXNldCUzRGRzKSUwQXJlc3VsdCUyMCUzRCUyMHRyYWluZXIudHJhaW4oKSUwQXByaW50X3N1bW1hcnkocmVzdWx0KQ==",highlighted:`training_args = TrainingArguments(
    per_device_train_batch_size=<span class="hljs-number">1</span>,
    gradient_accumulation_steps=<span class="hljs-number">4</span>,
    gradient_checkpointing=<span class="hljs-literal">True</span>,
    fp16=<span class="hljs-literal">True</span>,
    optim=<span class="hljs-string">&quot;adafactor&quot;</span>,
    **default_args,
)

trainer = Trainer(model=model, args=training_args, train_dataset=ds)
result = trainer.train()
print_summary(result)`}}),dl=new M({props:{code:"VGltZSUzQSUyMDU2LjU0JTBBU2FtcGxlcyUyRnNlY29uZCUzQSUyMDkuMDYlMEFHUFUlMjBtZW1vcnklMjBvY2N1cGllZCUzQSUyMDQ4NDclMjBNQi4=",highlighted:`<span class="hljs-keyword">Time:</span> 56.54
Samples/second: 9.06
GPU memory occupied: 4847 MB.`}}),ul=new v({}),yt=new X3({props:{$$slots:{default:[tA]},$$scope:{ctx:zr}}}),yl=new M({props:{code:"aW1wb3J0JTIwYml0c2FuZGJ5dGVzJTIwYXMlMjBibmIlMEFmcm9tJTIwdG9yY2glMjBpbXBvcnQlMjBubiUwQWZyb20lMjB0cmFuc2Zvcm1lcnMudHJhaW5lcl9wdF91dGlscyUyMGltcG9ydCUyMGdldF9wYXJhbWV0ZXJfbmFtZXMlMEElMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwVHJhaW5pbmdBcmd1bWVudHMocGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNENCUyQyUyMCoqZGVmYXVsdF9hcmdzKSUwQSUwQWRlY2F5X3BhcmFtZXRlcnMlMjAlM0QlMjBnZXRfcGFyYW1ldGVyX25hbWVzKG1vZGVsJTJDJTIwJTVCbm4uTGF5ZXJOb3JtJTVEKSUwQWRlY2F5X3BhcmFtZXRlcnMlMjAlM0QlMjAlNUJuYW1lJTIwZm9yJTIwbmFtZSUyMGluJTIwZGVjYXlfcGFyYW1ldGVycyUyMGlmJTIwJTIyYmlhcyUyMiUyMG5vdCUyMGluJTIwbmFtZSU1RCUwQW9wdGltaXplcl9ncm91cGVkX3BhcmFtZXRlcnMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwYXJhbXMlMjIlM0ElMjAlNUJwJTIwZm9yJTIwbiUyQyUyMHAlMjBpbiUyMG1vZGVsLm5hbWVkX3BhcmFtZXRlcnMoKSUyMGlmJTIwbiUyMGluJTIwZGVjYXlfcGFyYW1ldGVycyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMndlaWdodF9kZWNheSUyMiUzQSUyMHRyYWluaW5nX2FyZ3Mud2VpZ2h0X2RlY2F5JTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGFyYW1zJTIyJTNBJTIwJTVCcCUyMGZvciUyMG4lMkMlMjBwJTIwaW4lMjBtb2RlbC5uYW1lZF9wYXJhbWV0ZXJzKCklMjBpZiUyMG4lMjBub3QlMjBpbiUyMGRlY2F5X3BhcmFtZXRlcnMlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ3ZWlnaHRfZGVjYXklMjIlM0ElMjAwLjAlMkMlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElNUQlMEElMEFvcHRpbWl6ZXJfa3dhcmdzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIyYmV0YXMlMjIlM0ElMjAodHJhaW5pbmdfYXJncy5hZGFtX2JldGExJTJDJTIwdHJhaW5pbmdfYXJncy5hZGFtX2JldGEyKSUyQyUwQSUyMCUyMCUyMCUyMCUyMmVwcyUyMiUzQSUyMHRyYWluaW5nX2FyZ3MuYWRhbV9lcHNpbG9uJTJDJTBBJTdEJTBBb3B0aW1pemVyX2t3YXJncyU1QiUyMmxyJTIyJTVEJTIwJTNEJTIwdHJhaW5pbmdfYXJncy5sZWFybmluZ19yYXRlJTBBYWRhbV9ibmJfb3B0aW0lMjAlM0QlMjBibmIub3B0aW0uQWRhbThiaXQoJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyX2dyb3VwZWRfcGFyYW1ldGVycyUyQyUwQSUyMCUyMCUyMCUyMGJldGFzJTNEKHRyYWluaW5nX2FyZ3MuYWRhbV9iZXRhMSUyQyUyMHRyYWluaW5nX2FyZ3MuYWRhbV9iZXRhMiklMkMlMEElMjAlMjAlMjAlMjBlcHMlM0R0cmFpbmluZ19hcmdzLmFkYW1fZXBzaWxvbiUyQyUwQSUyMCUyMCUyMCUyMGxyJTNEdHJhaW5pbmdfYXJncy5sZWFybmluZ19yYXRlJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb
<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> nn
<span class="hljs-keyword">from</span> transformers.trainer_pt_utils <span class="hljs-keyword">import</span> get_parameter_names

training_args = TrainingArguments(per_device_train_batch_size=<span class="hljs-number">4</span>, **default_args)

decay_parameters = get_parameter_names(model, [nn.LayerNorm])
decay_parameters = [name <span class="hljs-keyword">for</span> name <span class="hljs-keyword">in</span> decay_parameters <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;bias&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> name]
optimizer_grouped_parameters = [
    {
        <span class="hljs-string">&quot;params&quot;</span>: [p <span class="hljs-keyword">for</span> n, p <span class="hljs-keyword">in</span> model.named_parameters() <span class="hljs-keyword">if</span> n <span class="hljs-keyword">in</span> decay_parameters],
        <span class="hljs-string">&quot;weight_decay&quot;</span>: training_args.weight_decay,
    },
    {
        <span class="hljs-string">&quot;params&quot;</span>: [p <span class="hljs-keyword">for</span> n, p <span class="hljs-keyword">in</span> model.named_parameters() <span class="hljs-keyword">if</span> n <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> decay_parameters],
        <span class="hljs-string">&quot;weight_decay&quot;</span>: <span class="hljs-number">0.0</span>,
    },
]

optimizer_kwargs = {
    <span class="hljs-string">&quot;betas&quot;</span>: (training_args.adam_beta1, training_args.adam_beta2),
    <span class="hljs-string">&quot;eps&quot;</span>: training_args.adam_epsilon,
}
optimizer_kwargs[<span class="hljs-string">&quot;lr&quot;</span>] = training_args.learning_rate
adam_bnb_optim = bnb.optim.Adam8bit(
    optimizer_grouped_parameters,
    betas=(training_args.adam_beta1, training_args.adam_beta2),
    eps=training_args.adam_epsilon,
    lr=training_args.learning_rate,
)`}}),bl=new M({props:{code:"dHJhaW5lciUyMCUzRCUyMFRyYWluZXIobW9kZWwlM0Rtb2RlbCUyQyUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTIwdHJhaW5fZGF0YXNldCUzRGRzJTJDJTIwb3B0aW1pemVycyUzRChhZGFtX2JuYl9vcHRpbSUyQyUyME5vbmUpKSUwQXJlc3VsdCUyMCUzRCUyMHRyYWluZXIudHJhaW4oKSUwQXByaW50X3N1bW1hcnkocmVzdWx0KQ==",highlighted:`trainer = Trainer(model=model, args=training_args, train_dataset=ds, optimizers=(adam_bnb_optim, <span class="hljs-literal">None</span>))
result = trainer.train()
print_summary(result)`}}),Ml=new M({props:{code:"VGltZSUzQSUyMDU1Ljk1JTBBU2FtcGxlcyUyRnNlY29uZCUzQSUyMDkuMTUlMEFHUFUlMjBtZW1vcnklMjBvY2N1cGllZCUzQSUyMDEzMDg1JTIwTUIu",highlighted:`<span class="hljs-keyword">Time:</span> 55.95
Samples/second: 9.15
GPU memory occupied: 13085 MB.`}}),vl=new M({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMkMlMEElMjAlMjAlMjAlMjBncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfY2hlY2twb2ludGluZyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBmcDE2JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCoqZGVmYXVsdF9hcmdzJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKG1vZGVsJTNEbW9kZWwlMkMlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUyMHRyYWluX2RhdGFzZXQlM0RkcyUyQyUyMG9wdGltaXplcnMlM0QoYWRhbV9ibmJfb3B0aW0lMkMlMjBOb25lKSklMEFyZXN1bHQlMjAlM0QlMjB0cmFpbmVyLnRyYWluKCklMEFwcmludF9zdW1tYXJ5KHJlc3VsdCk=",highlighted:`training_args = TrainingArguments(
    per_device_train_batch_size=<span class="hljs-number">1</span>,
    gradient_accumulation_steps=<span class="hljs-number">4</span>,
    gradient_checkpointing=<span class="hljs-literal">True</span>,
    fp16=<span class="hljs-literal">True</span>,
    **default_args,
)

trainer = Trainer(model=model, args=training_args, train_dataset=ds, optimizers=(adam_bnb_optim, <span class="hljs-literal">None</span>))
result = trainer.train()
print_summary(result)`}}),gl=new M({props:{code:"VGltZSUzQSUyMDQ5LjQ2JTBBU2FtcGxlcyUyRnNlY29uZCUzQSUyMDEwLjM1JTBBR1BVJTIwbWVtb3J5JTIwb2NjdXBpZWQlM0ElMjA1MzYzJTIwTUIu",highlighted:`<span class="hljs-keyword">Time:</span> 49.46
Samples/second: 10.35
GPU memory occupied: 5363 MB.`}}),Tl=new v({}),Ul=new v({}),_l=new M({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMkMlMEElMjAlMjAlMjAlMjBncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfY2hlY2twb2ludGluZyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBmcDE2JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCoqZGVmYXVsdF9hcmdzJTJDJTBBKQ==",highlighted:`training_args = TrainingArguments(
    per_device_train_batch_size=<span class="hljs-number">1</span>,
    gradient_accumulation_steps=<span class="hljs-number">4</span>,
    gradient_checkpointing=<span class="hljs-literal">True</span>,
    fp16=<span class="hljs-literal">True</span>,
    **default_args,
)`}}),Jl=new M({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjB0b3JjaC51dGlscy5kYXRhLmRhdGFsb2FkZXIlMjBpbXBvcnQlMjBEYXRhTG9hZGVyJTBBJTBBZGF0YWxvYWRlciUyMCUzRCUyMERhdGFMb2FkZXIoZHMlMkMlMjBiYXRjaF9zaXplJTNEdHJhaW5pbmdfYXJncy5wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUpJTBBJTBBaWYlMjB0cmFpbmluZ19hcmdzLmdyYWRpZW50X2NoZWNrcG9pbnRpbmclM0ElMEElMjAlMjAlMjAlMjBtb2RlbC5ncmFkaWVudF9jaGVja3BvaW50aW5nX2VuYWJsZSgpJTBBJTBBYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcihmcDE2JTNEdHJhaW5pbmdfYXJncy5mcDE2KSUwQW1vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwZGF0YWxvYWRlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUobW9kZWwlMkMlMjBhZGFtX2JuYl9vcHRpbSUyQyUyMGRhdGFsb2FkZXIpJTBBJTBBbW9kZWwudHJhaW4oKSUwQWZvciUyMHN0ZXAlMkMlMjBiYXRjaCUyMGluJTIwZW51bWVyYXRlKGRhdGFsb2FkZXIlMkMlMjBzdGFydCUzRDEpJTNBJTBBJTIwJTIwJTIwJTIwbG9zcyUyMCUzRCUyMG1vZGVsKCoqYmF0Y2gpLmxvc3MlMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zcyUyMCUyRiUyMHRyYWluaW5nX2FyZ3MuZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTBBJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IuYmFja3dhcmQobG9zcyklMEElMjAlMjAlMjAlMjBpZiUyMHN0ZXAlMjAlMjUlMjB0cmFpbmluZ19hcmdzLmdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUyMCUzRCUzRCUyMDAlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyLnplcm9fZ3JhZCgp",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator
<span class="hljs-keyword">from</span> torch.utils.data.dataloader <span class="hljs-keyword">import</span> DataLoader

dataloader = DataLoader(ds, batch_size=training_args.per_device_train_batch_size)

<span class="hljs-keyword">if</span> training_args.gradient_checkpointing:
    model.gradient_checkpointing_enable()

accelerator = Accelerator(fp16=training_args.fp16)
model, optimizer, dataloader = accelerator.prepare(model, adam_bnb_optim, dataloader)

model.train()
<span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(dataloader, start=<span class="hljs-number">1</span>):
    loss = model(**batch).loss
    loss = loss / training_args.gradient_accumulation_steps
    accelerator.backward(loss)
    <span class="hljs-keyword">if</span> step % training_args.gradient_accumulation_steps == <span class="hljs-number">0</span>:
        optimizer.step()
        optimizer.zero_grad()`}}),Il=new M({props:{code:"cHJpbnRfZ3B1X3V0aWxpemF0aW9uKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>print_gpu_utilization()
GPU memory occupied: <span class="hljs-number">5363</span> MB.`}}),$l=new v({}),zl=new v({}),Sl=new M({props:{code:"VHJhaW5pbmdBcmd1bWVudHMoZGVlcHNwZWVkJTNEJTIyJTJGcGF0aCUyRnRvJTJGZHNfY29uZmlnLmpzb24lMjIp",highlighted:'TrainingArguments(deepspeed=<span class="hljs-string">&quot;/path/to/ds_config.json&quot;</span>)'}}),Wl=new v({}),Bl=new v({}),Nl=new v({}),xl=new v({}),Yl=new v({}),tr=new v({}),ar=new v({}),ir=new M({props:{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC50b19iZXR0ZXJ0cmFuc2Zvcm1lcigp",highlighted:"model = model.to_bettertransformer()"}}),nr=new v({}),{c(){g=l("meta"),E=p(),U=l("h1"),$=l("a"),Mi=l("span"),c(Ot.$$.fragment),uu=p(),vi=l("span"),wu=s("Efficient Training on a Single GPU"),rp=p(),Be=l("p"),yu=s("This guide focuses on training large models efficiently on a single GPU. These approaches are still valid if you have access to a machine with multiple GPUs but you will also have access to additional methods outlined in the "),Cr=l("a"),bu=s("multi-GPU section"),Mu=s("."),op=p(),N=l("p"),vu=s("In this section we have a look at a few tricks to reduce the memory footprint and speed up training for large models and how they are integrated in the "),Gr=l("a"),gu=s("Trainer"),Tu=s(" and "),qt=l("a"),Uu=s("\u{1F917} Accelerate"),_u=s(". Each method can improve speed or memory usage which is summarized in the table below:"),sp=p(),Ne=l("table"),gi=l("thead"),le=l("tr"),Sr=l("th"),Ju=s("Method"),Eu=p(),Rr=l("th"),Au=s("Speed"),ju=p(),Pr=l("th"),Iu=s("Memory"),ku=p(),_=l("tbody"),re=l("tr"),Wr=l("td"),$u=s("Gradient accumulation"),zu=p(),Br=l("td"),Cu=s("No"),Gu=p(),Nr=l("td"),Su=s("Yes"),Ru=p(),oe=l("tr"),Qr=l("td"),Pu=s("Gradient checkpointing"),Wu=p(),Zr=l("td"),Bu=s("No"),Nu=p(),Fr=l("td"),Qu=s("Yes"),Zu=p(),se=l("tr"),xr=l("td"),Fu=s("Mixed precision training"),xu=p(),Yr=l("td"),Yu=s("Yes"),Lu=p(),Lr=l("td"),Du=s("(No)"),Xu=p(),ie=l("tr"),Dr=l("td"),Vu=s("Batch size"),Hu=p(),Xr=l("td"),Ou=s("Yes"),qu=p(),Vr=l("td"),Ku=s("Yes"),ew=p(),ne=l("tr"),Hr=l("td"),tw=s("Optimizer choice"),aw=p(),Or=l("td"),lw=s("Yes"),rw=p(),qr=l("td"),ow=s("Yes"),sw=p(),he=l("tr"),Kr=l("td"),iw=s("DataLoader"),nw=p(),eo=l("td"),hw=s("Yes"),pw=p(),to=l("td"),fw=s("No"),mw=p(),pe=l("tr"),ao=l("td"),cw=s("DeepSpeed Zero"),dw=p(),lo=l("td"),uw=s("No"),ww=p(),ro=l("td"),yw=s("Yes"),ip=p(),oo=l("p"),bw=s("A bracket means that it might not be strictly the case but is usually either not a main concern or negligible. Before we start make sure you have installed the following libraries:"),np=p(),c(Kt.$$.fragment),hp=p(),Q=l("p"),Mw=s("The "),Ti=l("code"),vw=s("nvidia-ml-py3"),gw=s(" library allows us to monitor the memory usage of the models from within Python. You might be familiar with the "),Ui=l("code"),Tw=s("nvidia-smi"),Uw=s(" command in the terminal - this library allows to access the same information in Python directly."),pp=p(),Qe=l("p"),_w=s("Then we create some dummy data. We create random token IDs between 100 and 30000 and binary labels for a classifier. In total we get 512 sequences each with length 512 and store them in a "),ea=l("a"),Jw=s("Dataset"),Ew=s(" with PyTorch format."),fp=p(),c(ta.$$.fragment),mp=p(),Ze=l("p"),Aw=s("We want to print some summary statistics for the GPU utilization and the training run with the "),so=l("a"),jw=s("Trainer"),Iw=s(". We setup a two helper functions to do just that:"),cp=p(),c(aa.$$.fragment),dp=p(),io=l("p"),kw=s("Let\u2019s verify that we start with a free GPU memory:"),up=p(),c(la.$$.fragment),wp=p(),no=l("p"),$w=s("That looks good: the GPU memory is not occupied as we would expect before we load any models. If that\u2019s not the case on your machine make sure to stop all processes that are using GPU memory. However, not all free GPU memory can be used by the user. When a model is loaded to the GPU also the kernels are loaded which can take up 1-2GB of memory. To see how much it is we load a tiny tensor into the GPU which triggers the kernels to be loaded as well."),yp=p(),c(ra.$$.fragment),bp=p(),ho=l("p"),zw=s("We see that the kernels alone take up 1.3GB of GPU memory. Now let\u2019s see how much space the model uses."),Mp=p(),fe=l("h2"),Fe=l("a"),_i=l("span"),c(oa.$$.fragment),Cw=p(),Ji=l("span"),Gw=s("Load Model"),vp=p(),xe=l("p"),Sw=s("First, we load the "),Ei=l("code"),Rw=s("bert-large-uncased"),Pw=s(" model. We load the model weights directly to the GPU so that we can check how much space just weights use."),gp=p(),c(sa.$$.fragment),Tp=p(),Ye=l("p"),Ww=s("We can see that the model weights alone take up 1.3 GB of the GPU memory. The exact number depends on the specific GPU you are using. Note that on newer GPUs a model can sometimes take up more space since the weights are loaded in an optimized fashion that speeds up the usage of the model. Now we can also quickly check if we get the same result as with "),Ai=l("code"),Bw=s("nvidia-smi"),Nw=s(" CLI:"),Up=p(),c(ia.$$.fragment),_p=p(),c(na.$$.fragment),Jp=p(),po=l("p"),Qw=s("We get the same number as before and you can also see that we are using a V100 GPU with 16GB of memory. So now we can start training the model and see how the GPU memory consumption changes. First, we set up a few standard training arguments that we will use across all our experiments:"),Ep=p(),c(ha.$$.fragment),Ap=p(),c(Le.$$.fragment),jp=p(),me=l("h2"),De=l("a"),ji=l("span"),c(pa.$$.fragment),Zw=p(),Ii=l("span"),Fw=s("Vanilla Training"),Ip=p(),Xe=l("p"),xw=s("As a first experiment we will use the "),fo=l("a"),Yw=s("Trainer"),Lw=s(" and train the model without any further modifications and a batch size of 4:"),kp=p(),c(fa.$$.fragment),$p=p(),c(ma.$$.fragment),zp=p(),mo=l("p"),Dw=s("We see that already a relatively small batch size almost fills up our GPU\u2019s entire memory. However, a larger batch size can often result in faster model convergence or better end performance. So ideally we want to tune the batch size to our model\u2019s needs and not to the GPU limitations. What\u2019s interesting is that we use much more memory than the size of the model. To understand a bit better why this is the case let\u2019s have look at a model\u2019s operations and memory needs."),Cp=p(),ce=l("h2"),Ve=l("a"),ki=l("span"),c(ca.$$.fragment),Xw=p(),$i=l("span"),Vw=s("Anatomy of Model's Operations"),Gp=p(),co=l("p"),Hw=s("Transformers architecture includes 3 main groups of operations grouped below by compute-intensity."),Sp=p(),Z=l("ol"),da=l("li"),zi=l("p"),Ci=l("strong"),Ow=s("Tensor Contractions"),qw=p(),ua=l("p"),Kw=s("Linear layers and components of Multi-Head Attention all do batched "),Gi=l("strong"),ey=s("matrix-matrix multiplications"),ty=s(". These operations are the most compute-intensive part of training a transformer."),ay=p(),wa=l("li"),Si=l("p"),Ri=l("strong"),ly=s("Statistical Normalizations"),ry=p(),ya=l("p"),oy=s("Softmax and layer normalization are less compute-intensive than tensor contractions, and involve one or more "),Pi=l("strong"),sy=s("reduction operations"),iy=s(", the result of which is then applied via a map."),ny=p(),ba=l("li"),Wi=l("p"),Bi=l("strong"),hy=s("Element-wise Operators"),py=p(),Ma=l("p"),fy=s("These are the remaining operators: "),Ni=l("strong"),my=s("biases, dropout, activations, and residual connections"),cy=s(". These are the least compute-intensive operations."),Rp=p(),uo=l("p"),dy=s("This knowledge can be helpful to know when analyzing performance bottlenecks."),Pp=p(),va=l("p"),uy=s("This summary is derived from "),ga=l("a"),wy=s("Data Movement Is All You Need: A Case Study on Optimizing Transformers 2020"),Wp=p(),de=l("h2"),He=l("a"),Qi=l("span"),c(Ta.$$.fragment),yy=p(),Zi=l("span"),by=s("Anatomy of Model's Memory"),Bp=s(`

We've seen that training the model uses much more memory than just putting the model on the GPU. This is because there are many components during training that use GPU memory. The components on GPU memory are the following:
1. model weights
2. optimizer states
3. gradients
4. forward activations saved for gradient computation
5. temporary buffers
6. functionality-specific memory
`),wo=l("p"),My=s("A typical model trained in mixed precision with AdamW requires 18 bytes per model parameter plus activation memory. For inference there are no optimizer states and gradients, so we can subtract those. And thus we end up with 6 bytes per model parameter for mixed precision inference, plus activation memory."),Np=p(),yo=l("p"),vy=s("Let\u2019s look at the details."),Qp=p(),bo=l("p"),Fi=l("strong"),gy=s("Model Weights:"),Zp=p(),Oe=l("ul"),xi=l("li"),Ty=s("4 bytes * number of parameters for fp32 training"),Uy=p(),Yi=l("li"),_y=s("6 bytes * number of parameters for mixed precision training (maintains a model in fp32 and one in fp16 in memory)"),Fp=p(),Mo=l("p"),Li=l("strong"),Jy=s("Optimizer States:"),xp=p(),F=l("ul"),Di=l("li"),Ey=s("8 bytes * number of parameters for normal AdamW (maintains 2 states)"),Ay=p(),vo=l("li"),jy=s("2 bytes * number of parameters for 8-bit AdamW optimizers like "),Ua=l("a"),Iy=s("bitsandbytes"),ky=p(),Xi=l("li"),$y=s("4 bytes * number of parameters for optimizers like SGD with momentum (maintains only 1 state)"),Yp=p(),go=l("p"),Vi=l("strong"),zy=s("Gradients"),Lp=p(),To=l("ul"),Hi=l("li"),Cy=s("4 bytes * number of parameters for either fp32 or mixed precision training (gradients are always kept in fp32)"),Dp=p(),Uo=l("p"),Oi=l("strong"),Gy=s("Forward Activations"),Xp=p(),_o=l("ul"),qi=l("li"),Sy=s("size depends on many factors, the key ones being sequence length, hidden size and batch size."),Vp=p(),Jo=l("p"),Ry=s("There are the input and output that are being passed and returned by the forward and the backward functions and the forward activations saved for gradient computation."),Hp=p(),Eo=l("p"),Ki=l("strong"),Py=s("Temporary Memory"),Op=p(),Ao=l("p"),Wy=s("Additionally there are all kinds of temporary variables which get released once the calculation is done, but in the moment these could require additional memory and could push to OOM. Therefore when coding it\u2019s crucial to think strategically about such temporary variables and sometimes to explicitly free those as soon as they are no longer needed."),qp=p(),jo=l("p"),en=l("strong"),By=s("Functionality-specific memory"),Kp=p(),Io=l("p"),Ny=s("Then your software could have special memory needs. For example, when generating text using beam search, the software needs to maintain multiple copies of inputs and outputs."),ef=p(),ko=l("p"),qe=l("strong"),tn=l("code"),Qy=s("forward"),Zy=s(" vs "),an=l("code"),Fy=s("backward"),xy=s(" Execution Speed"),tf=p(),$o=l("p"),Yy=s("For convolutions and linear layers there are 2x flops in the backward compared to the forward, which generally translates into ~2x slower (sometimes more, because sizes in the backward tend to be more awkward). Activations are usually bandwidth-limited, and it\u2019s typical for an activation to have to read more data in the backward than in the forward (e.g. activation forward reads once, writes once, activation backward reads twice, gradOutput and output of the forward, and writes once, gradInput)."),af=p(),zo=l("p"),Ly=s("So there are potentially a few places where we could save GPU memory or speed up operations. Let\u2019s start with a simple optimization: choosing the right batch size."),lf=p(),ue=l("h2"),Ke=l("a"),ln=l("span"),c(_a.$$.fragment),Dy=p(),rn=l("span"),Xy=s("Batch sizes"),rf=p(),Co=l("p"),Vy=s("One gets the most efficient performance when batch sizes and input/output neuron counts are divisible by a certain number, which typically starts at 8, but can be much higher as well. That number varies a lot depending on the specific hardware being used and the dtype of the model."),of=p(),x=l("p"),Hy=s("For example for fully connected layers (which correspond to GEMMs), NVIDIA provides recommendations for "),Ja=l("a"),Oy=s("input/output neuron counts"),qy=s(" and "),Ea=l("a"),Ky=s("batch size"),eb=s("."),sf=p(),Aa=l("p"),ja=l("a"),tb=s("Tensor Core Requirements"),ab=s(" define the multiplier based on the dtype and the hardware. For example, for fp16 a multiple of 8 is recommended, but on A100 it\u2019s 64!"),nf=p(),et=l("p"),lb=s("For parameters that are small, there is also "),Ia=l("a"),rb=s("Dimension Quantization Effects"),ob=s(" to consider, this is where tiling happens and the right multiplier can have a significant speedup."),hf=p(),we=l("h2"),tt=l("a"),on=l("span"),c(ka.$$.fragment),sb=p(),sn=l("span"),ib=s("Gradient Accumulation"),pf=p(),Go=l("p"),nb=s("The idea behind gradient accumulation is to instead of calculating the gradients for the whole batch at once to do it in smaller steps. The way we do that is to calculate the gradients iteratively in smaller batches by doing a forward and backward pass through the model and accumulating the gradients in the process. When enough gradients are accumulated we run the model\u2019s optimization step. This way we can easily increase the overall batch size to numbers that would never fit into the GPU\u2019s memory. In turn, however, the added forward and backward passes can slow down the training a bit."),ff=p(),z=l("p"),hb=s("We can use gradient accumulation in the "),So=l("a"),pb=s("Trainer"),fb=s(" by simply adding the "),nn=l("code"),mb=s("gradient_accumulation_steps"),cb=s(" argument to "),Ro=l("a"),db=s("TrainingArguments"),ub=s(". Let\u2019s see how it impacts the models memory footprint:"),mf=p(),c($a.$$.fragment),cf=p(),c(za.$$.fragment),df=p(),A=l("p"),wb=s("We can see that the memory footprint was dramatically reduced at the cost of being only slightly slower than the vanilla run. Of course, this would change as you increase the number of accumulation steps. In general you would want to max out the GPU usage as much as possible. So in our case, the batch_size of 4 was already pretty close to the GPU\u2019s limit. If we wanted to train with a batch size of 64 we should not use "),hn=l("code"),yb=s("per_device_train_batch_size=1"),bb=s(" and "),pn=l("code"),Mb=s("gradient_accumulation_steps=64"),vb=s(" but instead "),fn=l("code"),gb=s("per_device_train_batch_size=4"),Tb=s(" and "),mn=l("code"),Ub=s("gradient_accumulation_steps=16"),_b=s(" which has the same effective batch size while making better use of the available GPU resources."),uf=p(),Y=l("p"),Jb=s("For more details see the benchmarks for "),Ca=l("a"),Eb=s("RTX-3090"),Ab=s(`
and `),Ga=l("a"),jb=s("A100"),Ib=s("."),wf=p(),Po=l("p"),kb=s("Next we have a look at another trick to save a little bit more GPU memory called gradient checkpointing."),yf=p(),ye=l("h2"),at=l("a"),cn=l("span"),c(Sa.$$.fragment),$b=p(),dn=l("span"),zb=s("Gradient Checkpointing"),bf=p(),Wo=l("p"),Cb=s("Even when we set the batch size to 1 and use gradient accumulation we can still run out of memory when working with large models. In order to compute the gradients during the backward pass all activations from the forward pass are normally saved. This can create a big memory overhead. Alternatively, one could forget all activations during the forward pass and recompute them on demand during the backward pass. This would however add a significant computational overhead and slow down training."),Mf=p(),lt=l("p"),Gb=s("Gradient checkpointing strikes a compromise between the two approaches and saves strategically selected activations throughout the computational graph so only a fraction of the activations need to be re-computed for the gradients. See "),Ra=l("a"),Sb=s("this great article"),Rb=s(" explaining the ideas behind gradient checkpointing."),vf=p(),L=l("p"),Pb=s("To enable gradient checkpointing in the "),Bo=l("a"),Wb=s("Trainer"),Bb=s(" we only need to pass it as a flag to the "),No=l("a"),Nb=s("TrainingArguments"),Qb=s(". Everything else is handled under the hood:"),gf=p(),c(Pa.$$.fragment),Tf=p(),c(Wa.$$.fragment),Uf=p(),Qo=l("p"),Zb=s("We can see that this saved some more memory but at the same time training became a bit slower. A general rule of thumb is that gradient checkpointing slows down training by about 20%. Let\u2019s have a look at another method with which we can regain some speed: mixed precision training."),_f=p(),be=l("h2"),rt=l("a"),un=l("span"),c(Ba.$$.fragment),Fb=p(),wn=l("span"),xb=s("Floating Data Types"),Jf=p(),Zo=l("p"),Yb=s("The idea of mixed precision training is that not all variables need to be stored in full (32-bit) floating point precision. If we can reduce the precision the variables and their computations are faster. Here are the commonly used floating point data types choice of which impacts both memory usage and throughput:"),Ef=p(),C=l("ul"),Na=l("li"),Lb=s("fp32 ("),yn=l("code"),Db=s("float32"),Xb=s(")"),Vb=p(),Qa=l("li"),Hb=s("fp16 ("),bn=l("code"),Ob=s("float16"),qb=s(")"),Kb=p(),Za=l("li"),eM=s("bf16 ("),Mn=l("code"),tM=s("bfloat16"),aM=s(")"),lM=p(),vn=l("li"),rM=s("tf32 (CUDA internal data type)"),Af=p(),Fo=l("p"),oM=s("Here is a diagram that shows how these data types correlate to each other."),jf=p(),Me=l("p"),xo=l("img"),sM=s(`
(source: `),Fa=l("a"),iM=s("NVIDIA Blog"),nM=s(")"),If=p(),Yo=l("p"),hM=s("While fp16 and fp32 have been around for quite some time, bf16 and tf32 are only available on the Ampere architecture GPUS and TPUs support bf16 as well. Let\u2019s start with the most commonly used method which is FP16 training/"),kf=p(),ve=l("h3"),ot=l("a"),gn=l("span"),c(xa.$$.fragment),pM=p(),Tn=l("span"),fM=s("FP16 Training"),$f=p(),D=l("p"),mM=s("The idea of mixed precision training is that not all variables need to be stored in full (32-bit) floating point precision. If we can reduce the precision the variales and their computations are faster. The main advantage comes from saving the activations in half (16-bit) precision. Although the gradients are also computed in half precision they are converted back to full precision for the optimization step so no memory is saved here. Since the model is present on the GPU in both 16-bit and 32-bit precision this can use more GPU memory (1.5x the original model is on the GPU), especially for small batch sizes. Since some computations are performed in full and some in half precision this approach is also called mixed precision training. Enabling mixed precision training is also just a matter of setting the "),Un=l("code"),cM=s("fp16"),dM=s(" flag to "),_n=l("code"),uM=s("True"),wM=s(":"),zf=p(),c(Ya.$$.fragment),Cf=p(),c(La.$$.fragment),Gf=p(),Lo=l("p"),yM=s("We can see that this is almost twice as fast as the vanilla training. Let\u2019s add it to the mix of the previous methods:"),Sf=p(),c(Da.$$.fragment),Rf=p(),c(Xa.$$.fragment),Pf=p(),Do=l("p"),bM=s("We can see that with these tweaks we use about half the GPU memory as at the beginning while also being slightly faster."),Wf=p(),ge=l("h3"),st=l("a"),Jn=l("span"),c(Va.$$.fragment),MM=p(),En=l("span"),vM=s("BF16"),Bf=s("\n\nIf you have access to a Ampere or newer hardware you can use bf16 for your training and evaluation. While bf16 has a worse precision than fp16, it has a much much bigger dynamic range. Therefore, if in the past you were experiencing overflow issues while training the model, bf16 will prevent this from happening most of the time. Remember that in fp16 the biggest number you can have is `65535` and any number above that will overflow. A bf16 number can be as large as `3.39e+38` (!) which is about the same as fp32 - because both have 8-bits used for the numerical range.\n"),Xo=l("p"),gM=s("You can enable BF16 in the \u{1F917} Trainer with:"),Nf=p(),c(Ha.$$.fragment),Qf=p(),Te=l("h3"),it=l("a"),An=l("span"),c(Oa.$$.fragment),TM=p(),jn=l("span"),UM=s("TF32"),Zf=s(`

The Ampere hardware uses a magical data type called tf32. It has the same numerical range as fp32 (8-bits), but instead of 23 bits precision it has only 10 bits (same as fp16) and uses only 19 bits in total.
`),Vo=l("p"),_M=s("It\u2019s magical in the sense that you can use the normal fp32 training and/or inference code and by enabling tf32 support you can get up to 3x throughput improvement. All you need to do is to add this to your code:"),Ff=p(),c(qa.$$.fragment),xf=p(),Ho=l("p"),JM=s("When this is done CUDA will automatically switch to using tf32 instead of fp32 where it\u2019s possible. This, of course, assumes that the used GPU is from the Ampere series."),Yf=p(),nt=l("p"),EM=s("Like all cases with reduced precision this may or may not be satisfactory for your needs, so you have to experiment and see. According to "),Ka=l("a"),AM=s("NVIDIA research"),jM=s(" the majority of machine learning training shouldn\u2019t be impacted and showed the same perplexity and convergence as the fp32 training."),Lf=p(),Oo=l("p"),IM=s("If you\u2019re already using fp16 or bf16 mixed precision it may help with the throughput as well."),Df=p(),qo=l("p"),kM=s("You can enable this mode in the \u{1F917} Trainer with:"),Xf=p(),c(el.$$.fragment),Vf=p(),Ko=l("p"),$M=s("By default the PyTorch default is used."),Hf=p(),X=l("p"),zM=s("Note: tf32 mode is internal to CUDA and can\u2019t be accessed directly via "),In=l("code"),CM=s("tensor.to(dtype=torch.tf32)"),GM=s(" as "),kn=l("code"),SM=s("torch.tf32"),RM=s(" doesn\u2019t exist."),Of=p(),ht=l("p"),PM=s("Note: you need "),$n=l("code"),WM=s("torch>=1.7"),BM=s(" to enjoy this feature."),qf=p(),V=l("p"),NM=s(`You can also see a variety of benchmarks on tf32 vs other precisions:
`),tl=l("a"),QM=s("RTX-3090"),ZM=s(` and
`),al=l("a"),FM=s("A100"),xM=s("."),Kf=p(),es=l("p"),YM=s("We\u2019ve now seen how we can change the floating types to increase throughput, but we are not done, yet! There is another area where we can save GPU memory: the optimizer."),em=p(),Ue=l("h2"),pt=l("a"),zn=l("span"),c(ll.$$.fragment),LM=p(),Cn=l("span"),DM=s("Optimizer"),tm=p(),ts=l("p"),XM=s("The most common optimizer used to train transformer model is Adam or AdamW (Adam with weight decay). Adam achieves good convergence by storing the rolling average of the previous gradients which, however, adds an additional memory footprint of the order of the number of model parameters. One remedy to this is to use an alternative optimizer such as Adafactor, which works well for some models but often it has instability issues."),am=p(),ft=l("p"),VM=s("HF Trainer integrates a variety of optimisers that can be used out of box. To activate the desired optimizer simply pass the "),Gn=l("code"),HM=s("--optim"),OM=s(" flag to the command line."),lm=p(),as=l("p"),qM=s("To see which optimizers are currently supported:"),rm=p(),c(rl.$$.fragment),om=p(),H=l("p"),KM=s("For example, if you have "),ol=l("a"),ev=s("NVIDIA/apex"),tv=s(" installed "),Sn=l("code"),av=s("--optim adamw_apex_fused"),lv=s(" will give you the fastest training experience among all supported AdamW optimizers."),sm=p(),mt=l("p"),rv=s("On the other hand "),sl=l("a"),ov=s("8bit BNB optimizer"),sv=s(" can save 3/4 of memory normally used by a typical AdamW optimizer if it is configured to quantize all optimizer states, but in some situations only some optimizer states are quintized and then more memory is used."),im=p(),ct=l("p"),iv=s("Let\u2019s get a feel for the numbers and use for example use a 3B-parameter model, like "),Rn=l("code"),nv=s("t5-3b"),hv=s(". Note that since a Gigabyte correpsonds to a billion bytes we can simply multiply the parameters (in billions) with the number of necessary bytes per parameter to get Gigabytes of GPU memory usage:"),nm=p(),O=l("ul"),il=l("li"),pv=s("A standard AdamW uses 8 bytes for each parameter, here the optimizer will need ("),Pn=l("code"),fv=s("8*3"),mv=s(") 24GB of GPU memory."),cv=p(),nl=l("li"),dv=s("Adafactor uses slightly more than 4 bytes, so ("),Wn=l("code"),uv=s("4*3"),wv=s(") 12GB and then some extra."),yv=p(),hl=l("li"),bv=s("8bit BNB quantized optimizer will use only ("),Bn=l("code"),Mv=s("2*3"),vv=s(") 6GB if all optimizer states are quantized."),hm=p(),ls=l("p"),gv=s("Let\u2019s have a look at Adafactor first."),pm=p(),_e=l("h3"),dt=l("a"),Nn=l("span"),c(pl.$$.fragment),Tv=p(),Qn=l("span"),Uv=s("Adafactor"),fm=p(),ut=l("p"),_v=s("Instead of keeping the rolling average for each element in the weight matrices Adafactor only stores aggregated information (row- and column-wise sums of the rolling averages) which reduces the footprint considerably. One downside of Adafactor is that in some instances convergence can be slower than Adam\u2019s so some experimentation is advised here. We can use Adafactor simply by setting "),Zn=l("code"),Jv=s('optim="adafactor"'),Ev=s(":"),mm=p(),c(fl.$$.fragment),cm=p(),c(ml.$$.fragment),dm=p(),rs=l("p"),Av=s("We can see that this saves a few more GB on the GPU. Let\u2019s see how it looks when we add it to the other methods we introduced earlier:"),um=p(),c(cl.$$.fragment),wm=p(),c(dl.$$.fragment),ym=p(),os=l("p"),jv=s("We went from 15 GB memory usage to 5 GB - a 3x improvement while maintaining the throughput! However, as mentioned before, the convergence of Adafactor can be worse than Adam. There is an alternative to Adafactor called 8-bit Adam that takes a slightly different approach."),bm=p(),Je=l("h3"),wt=l("a"),Fn=l("span"),c(ul.$$.fragment),Iv=p(),xn=l("span"),kv=s("8-bit Adam"),Mm=p(),ss=l("p"),$v=s("Instead of aggregating optimizer states like Adafactor, 8-bit Adam keeps the full state and quantizes it. Quantization means that it stores the state with lower precision and dequantizes it only for the optimization. This is similar to the idea behind FP16 training where using variables with lower precision saves memory."),vm=p(),j=l("p"),zv=s("In contrast to the previous approaches is this one not integrated into the "),is=l("a"),Cv=s("Trainer"),Gv=s(" as a simple flag. We need to install the 8-bit optimizer and then pass it as a custom optimizer to the "),ns=l("a"),Sv=s("Trainer"),Rv=s(". Follow the installation guide in the Github "),wl=l("a"),Pv=s("repo"),Wv=s(" to install the "),Yn=l("code"),Bv=s("bitsandbytes"),Nv=s(" library that implements the 8-bit Adam optimizer."),gm=p(),hs=l("p"),Qv=s("Once installed, we just need to initialize the the optimizer. Although this looks like a considerable amount of work it actually just involves two steps: first we need to group the model\u2019s parameters into two groups where to one group we apply weight decay and to the other we don\u2019t. Usually, biases and layer norm parameters are not weight decayed. Then in a second step we just do some argument housekeeping to use the same parameters as the previously used AdamW optimizer."),Tm=p(),c(yt.$$.fragment),Um=p(),c(yl.$$.fragment),_m=p(),bt=l("p"),Zv=s("We can now pass the custom optimizer as an argument to the "),Ln=l("code"),Fv=s("Trainer"),xv=s(":"),Jm=p(),c(bl.$$.fragment),Em=p(),c(Ml.$$.fragment),Am=p(),ps=l("p"),Yv=s("We can see that we get a similar memory improvement as with Adafactor while keeping the full rolling average of the gradients. Let\u2019s repeat the experiment with the full settings:"),jm=p(),c(vl.$$.fragment),Im=p(),c(gl.$$.fragment),km=p(),fs=l("p"),Lv=s("Again, we get about a 3x memory improvement and even slightly higher throughput as using Adafactor. So we have seen how we can optimize the memory footprint of large models. The following plot summarizes all our experiments:"),$m=p(),ms=l("p"),cs=l("img"),zm=p(),Ee=l("h3"),Mt=l("a"),Dn=l("span"),c(Tl.$$.fragment),Dv=p(),Xn=l("span"),Vn=l("code"),Xv=s("_multi_tensor"),Cm=s(`

pytorch-nightly introduced \`torch.optim._multi_tensor\` which should significantly speed up the optimizers for situations with lots of small feature tensors. It should eventually become the default, but if you want to experiment with it sooner and don't mind using the bleed-edge, see: https://github.com/huggingface/transformers/issues/9965
`),Ae=l("h2"),vt=l("a"),Hn=l("span"),c(Ul.$$.fragment),Vv=p(),On=l("span"),Hv=s("Using \u{1F917} Accelerate"),Gm=p(),q=l("p"),Ov=s("So far we have used the "),ds=l("a"),qv=s("Trainer"),Kv=s(" to run the experiments but a more flexible alternative to that approach is to use \u{1F917} Accelerate. With \u{1F917} Accelerate you have full control over the training loop and can essentially write the loop in pure PyTorch with some minor modifications. In turn it allows you to easily scale across different infrastructures such as CPUs, GPUs, TPUs, or distributed multi-GPU setups without changing any code. Let\u2019s see what it takes to implement all of the above tweaks in \u{1F917} Accelerate. We can still use the "),us=l("a"),eg=s("TrainingArguments"),tg=s(" to wrap the training settings:"),Sm=p(),c(_l.$$.fragment),Rm=p(),ws=l("p"),ag=s("The full example training loop with \u{1F917} Accelerate is only a handful of lines of code long:"),Pm=p(),c(Jl.$$.fragment),Wm=p(),J=l("p"),lg=s("First we wrap the dataset in a "),El=l("a"),qn=l("code"),rg=s("DataLoader"),og=s(". Then we can enable gradient checkpointing by calling the model\u2019s "),ys=l("a"),sg=s("gradient_checkpointing_enable()"),ig=s(" method. When we initialize the "),Al=l("a"),Kn=l("code"),ng=s("Accelerator"),hg=s(" we can specify if we want to use mixed precision training and it will take care of it for us in the "),eh=l("code"),pg=s("prepare"),fg=s(" call. During the "),jl=l("a"),th=l("code"),mg=s("prepare"),cg=s(" call the dataloader will also be distributed across workers should we use multiple GPUs. We use the same 8-bit optimizer from the earlier experiments."),Bm=p(),gt=l("p"),dg=s("Finally, we can write the main training loop. Note that the "),ah=l("code"),ug=s("backward"),wg=s(" call is handled by \u{1F917} Accelerate. We can also see how gradient accumulation works: we normalize the loss so we get the average at the end of accumulation and once we have enough steps we run the optimization. Now the question is: does this use the same amount of memory as the previous steps? Let\u2019s check:"),Nm=p(),c(Il.$$.fragment),Qm=p(),Tt=l("p"),yg=s("Indeed it does. Implementing these optimization techniques with \u{1F917} Accelerate only takes a handful of lines of code and comes with the benefit of more flexiblity in the training loop. For a full documentation of all features have a look at the "),kl=l("a"),bg=s("Accelerate documentation"),Mg=s("."),Zm=p(),je=l("h2"),Ut=l("a"),lh=l("span"),c($l.$$.fragment),vg=p(),rh=l("span"),gg=s("DataLoader"),Fm=p(),bs=l("p"),Tg=s("One of the important requirements to reach great training speed is the ability to feed the GPU at the maximum speed it can handle. By default everything happens in the main process and it might not be able to read the data from disk fast enough, and thus create a bottleneck, leading to GPU under-utilization."),xm=p(),_t=l("ul"),Ms=l("li"),oh=l("code"),Ug=s("DataLoader(pin_memory=True, ...)"),_g=s(" which ensures that the data gets preloaded into the pinned memory on CPU and typically leads to much faster transfers from CPU to GPU memory."),Jg=p(),vs=l("li"),sh=l("code"),Eg=s("DataLoader(num_workers=4, ...)"),Ag=s(" - spawn several workers to pre-load data faster - during training watch the GPU utilization stats and if it\u2019s far from 100% experiment with raising the number of workers. Of course, the problem could be elsewhere so a very big number of workers won\u2019t necessarily lead to a better performance."),Ym=p(),Ie=l("h2"),Jt=l("a"),ih=l("span"),c(zl.$$.fragment),jg=p(),nh=l("span"),Ig=s("DeepSpeed ZeRO"),Lm=p(),Et=l("p"),kg=s("The in-depth details on how to use Deepspeed can be found "),gs=l("a"),$g=s("here"),zg=s("."),Dm=p(),Ts=l("p"),Cg=s("First, a quick decision tree:"),Xm=p(),At=l("ol"),hh=l("li"),Gg=s("Model fits onto a single GPU and you have enough space to fit a small batch size - you don\u2019t need to use Deepspeed as it\u2019ll only slow things down in this use case."),Sg=p(),ph=l("li"),Rg=s("Model doesn\u2019t fit onto a single GPU or you can\u2019t fit a small batch - use DeepSpeed ZeRO + CPU Offload and for much larger models NVMe Offload."),Vm=p(),jt=l("p"),Pg=s("Now if the decision tree suggested you use DeepSpeed first you need to "),Us=l("a"),Wg=s("install it"),Bg=s(", then follow one of the following guides to create a configuration file and launch DeepSpeed."),Hm=p(),_s=l("p"),Ng=s("Activation:"),Om=p(),G=l("ul"),fh=l("li"),Cl=l("p"),Qg=s("HF Trainer-based examples: see this "),Js=l("a"),Zg=s("guide"),Fg=s("."),xg=p(),Gl=l("li"),mh=l("p"),Yg=s("Custom HF Trainer-based program: Same as above, but pass:"),Lg=p(),c(Sl.$$.fragment),Dg=p(),ch=l("li"),Rl=l("p"),Xg=s("Deployment in Notebooks: see this "),Es=l("a"),Vg=s("guide"),Hg=s("."),Og=p(),dh=l("li"),ke=l("p"),qg=s("Custom training loop: This is somewhat complex but you can study how this is implemented in "),Pl=l("a"),Kg=s("HF Trainer"),e1=s(" - simply search for "),uh=l("code"),t1=s("deepspeed"),a1=s(" in the code."),qm=p(),$e=l("h2"),It=l("a"),wh=l("span"),c(Wl.$$.fragment),l1=p(),yh=l("span"),r1=s("Choice of GPU"),Km=s(`

Sometimes, even when applying all the above tweaks the throughput on a given GPU might still not be good enough. One easy solution is to change the type of GPU. For example switching from let's say a K80 (which you typically get on Google Colab) to a fancier GPU such as the V100 or A100. Although they are more expensive they are usually more cost effective than cheaper GPUs due to their larger memory and faster architecture.
`),As=l("p"),o1=s("Now, let\u2019s take a step back and discuss what we should optimize for when scaling the training of large models."),ec=p(),ze=l("h2"),kt=l("a"),bh=l("span"),c(Bl.$$.fragment),s1=p(),Mh=l("span"),i1=s("How to scale"),tc=p(),js=l("p"),n1=s("When we train models there are a two aspects we want to optimize at the same time:"),ac=p(),$t=l("ul"),vh=l("li"),h1=s("Data throughput/training time"),p1=p(),gh=l("li"),f1=s("Model performance"),lc=p(),Is=l("p"),m1=s("We have seen that each method changes the memory usage and throughput. In general we want to maximize the throughput (samples/second) to minimize the training cost. This is generally achieved by utilizing the GPU as much as possible and thus filling GPU memory to its limit. For example, as mentioned earlier, we only employ gradient accumulation when we want to use a batch size beyond the size of the GPU memory. If the desired batch size fits into memory then there is no reason to apply gradient accumulation which will only slow down training."),rc=p(),ks=l("p"),c1=s("The second objective is model performance. Just because we can does not mean we should use a large batch size. As part of hyperparameter tuning you should determine which batch size yields the best result and then optimize the throughput accordingly."),oc=p(),Ce=l("h2"),zt=l("a"),Th=l("span"),c(Nl.$$.fragment),d1=p(),Uh=l("span"),u1=s("Efficient Software Prebuilds"),sc=p(),Ct=l("p"),w1=s("PyTorch\u2019s "),Ql=l("a"),y1=s("pip and conda builds"),b1=s(" come prebuit with the cuda toolkit which is enough to run PyTorch, but it is insufficient if you need to build cuda extensions."),ic=p(),Gt=l("p"),M1=s("At times it may take an additional effort to pre-build some components, e.g., if you\u2019re using libraries like "),_h=l("code"),v1=s("apex"),g1=s(" that don\u2019t come pre-compiled. In other situations figuring out how to install the right cuda toolkit system-wide can be complicated. To address these users\u2019 needs PyTorch and NVIDIA release a new version of NGC docker container which already comes with everything prebuilt and you just need to install your programs on it and it will run out of the box."),nc=p(),$s=l("p"),T1=s("This approach is also useful if you want to tweak the pytorch source and/or make a new customized build."),hc=p(),K=l("p"),U1=s("To find the docker image version you want start "),Zl=l("a"),_1=s("here"),J1=s(", choose one of the latest monthly releases. Go into the release\u2019s notes for the desired release, check that the environment\u2019s components are matching your needs (including NVIDIA Driver requirements!) and then at the very top of that document go to the corresponding NGC page. If for some reason you get lost, here is "),Fl=l("a"),E1=s("the index of all PyTorch NGC images"),A1=s("."),pc=p(),zs=l("p"),j1=s("Next follow the instructions to download and deploy the docker image."),fc=p(),Ge=l("h2"),St=l("a"),Jh=l("span"),c(xl.$$.fragment),I1=p(),Eh=l("span"),k1=s("Sparsity"),mc=p(),Se=l("h3"),Rt=l("a"),Ah=l("span"),c(Yl.$$.fragment),$1=p(),jh=l("span"),z1=s("Mixture of Experts"),cc=p(),Cs=l("p"),C1=s(`Quite a few of the recent papers reported a 4-5x training speedup and a faster inference by integrating
Mixture of Experts (MoE) into the Transformer models.`),dc=p(),Gs=l("p"),G1=s("Since it has been discovered that more parameters lead to better performance, this technique allows to increase the number of parameters by an order of magnitude without increasing training costs."),uc=p(),Ss=l("p"),S1=s("In this approach every other FFN layer is replaced with a MoE Layer which consists of many experts, with a gated function that trains each expert in a balanced way depending on the input token\u2019s position in a sequence."),wc=p(),Rs=l("p"),Ps=l("img"),yc=p(),Pt=l("p"),R1=s("(source: "),Ll=l("a"),P1=s("GLAM"),W1=s(")"),bc=p(),Ws=l("p"),B1=s("You can find exhaustive details and comparison tables in the papers listed at the end of this section."),Mc=p(),Bs=l("p"),N1=s("The main drawback of this approach is that it requires staggering amounts of GPU memory - almost an order of magnitude larger than its dense equivalent. Various distillation and approaches are proposed to how to overcome the much higher memory requirements."),vc=p(),Ns=l("p"),Q1=s("There is direct trade-off though, you can use just a few experts with a 2-3x smaller base model instead of dozens or hundreds experts leading to a 5x smaller model and thus increase the training speed moderately while increasing the memory requirements moderately as well."),gc=p(),Qs=l("p"),Z1=s("Most related papers and implementations are built around Tensorflow/TPUs:"),Tc=p(),ee=l("ul"),Ih=l("li"),Dl=l("a"),F1=s("GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding"),x1=p(),kh=l("li"),Xl=l("a"),Y1=s("Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity"),L1=p(),$h=l("li"),Vl=l("a"),D1=s("GLaM: Generalist Language Model (GLaM)"),Uc=p(),T=l("p"),X1=s("And for Pytorch DeepSpeed has built one as well: "),Hl=l("a"),V1=s("DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale"),H1=s(", "),Ol=l("a"),O1=s("Mixture of Experts"),q1=s(" - blog posts:  "),ql=l("a"),K1=s("1"),eT=s(", "),Kl=l("a"),tT=s("2"),aT=s(" and specific deployment with large transformer-based natural language generation models: "),er=l("a"),lT=s("blog post"),rT=s(", "),Zs=l("a"),oT=s("Megatron-Deepspeed branch"),sT=s("."),_c=p(),Re=l("h2"),Wt=l("a"),zh=l("span"),c(tr.$$.fragment),iT=p(),Ch=l("span"),nT=s("Scaling beyond a single GPU"),Jc=p(),Fs=l("p"),hT=s("For some applications, such as pretraining large language models, applying all the approaches above might still not be fast enough. In this case you want to scale your experiment to several GPUs."),Ec=p(),Bt=l("p"),pT=s("Another use case for training on many GPUs is if the model does not fit on a single GPU with all the mentioned tricks. There are still more methods we can apply although life starts to get a bit more complicated. This usually involves some form of pipeline or tensor parallelism where the model itself is distributed across several GPUs. One can also make use of DeepSpeed which implements some of these parallelism strategies along with some more optimization to reduce the memory footprint such as partitioning the optimizer states. You can read more about this in the "),xs=l("a"),fT=s("\u201CMulti-GPU training\u201D section"),mT=s("."),Ac=p(),Pe=l("h2"),Nt=l("a"),Gh=l("span"),c(ar.$$.fragment),cT=p(),Sh=l("span"),dT=s("Using PyTorch native attention"),jc=p(),S=l("p"),uT=s("PyTorch 2.0 released the native "),lr=l("a"),Rh=l("code"),wT=s("torch.nn.functional.scaled_dot_product_attention"),yT=s(" (SDPA), that allows to use fused GPU kernels as "),rr=l("a"),bT=s("memory-efficient attention"),MT=s(" and "),or=l("a"),vT=s("flash attention"),gT=s("."),Ic=p(),Qt=l("p"),TT=s("After installing the "),sr=l("a"),Ph=l("code"),UT=s("optimum"),_T=s(" package, the relevant internal modules can be replaced to use PyTorch\u2019s native attention with:"),kc=p(),c(ir.$$.fragment),$c=p(),Ys=l("p"),JT=s("Training can then be done as usual."),zc=p(),We=l("h2"),Zt=l("a"),Wh=l("span"),c(nr.$$.fragment),ET=p(),Bh=l("span"),AT=s("Using torch.compile"),Cc=p(),Ft=l("p"),jT=s("PyTorch 2.0 introduces a new compile function, you can learn more about it "),hr=l("a"),IT=s("in their documentation"),kT=s(". It uses Python\u2019s frame evaluation API to automatically create a graph from existing PyTorch programs. After capturing the graph, different backends can be deployed to lower the graph to an optimized engine. You can choose one option below for performance boost."),Gc=p(),B=l("p"),Nh=l("code"),$T=s("torch.compile"),zT=s(" has a growing list of backends, which can be found in "),pr=l("a"),CT=s("backends.py"),GT=s(`
or `),Qh=l("code"),ST=s("torchdynamo.list_backends()"),RT=s(" each of which with its optional dependencies."),Sc=p(),Ls=l("p"),PT=s("Some of the most commonly used backends are"),Rc=p(),fr=l("p"),Zh=l("strong"),WT=s("Debugging backends"),BT=s(":"),Pc=p(),xt=l("ul"),Ds=l("li"),Fh=l("code"),NT=s('dynamo.optimize("eager")'),QT=s(" - Uses PyTorch to run the extracted GraphModule. This is quite useful in debugging TorchDynamo issues."),ZT=p(),Xs=l("li"),xh=l("code"),FT=s('dynamo.optimize("aot_eager")'),xT=s(" - Uses AotAutograd with no compiler, i.e, just using PyTorch eager for the AotAutograd\u2019s extracted forward and backward graphs. This is useful for debugging, and unlikely to give speedups."),Wc=p(),mr=l("p"),Yh=l("strong"),YT=s("Training & inference backends"),LT=s(":"),Bc=p(),R=l("ul"),cr=l("li"),Lh=l("code"),DT=s('dynamo.optimize("inductor")'),XT=s(" - Uses TorchInductor backend with AotAutograd and cudagraphs by leveraging codegened Triton kernels  "),dr=l("a"),VT=s("Read more"),HT=p(),ur=l("li"),Dh=l("code"),OT=s('dynamo.optimize("nvfuser")'),qT=s(" -  nvFuser with TorchScript. "),wr=l("a"),KT=s("Read more"),e2=p(),yr=l("li"),Xh=l("code"),t2=s('dynamo.optimize("aot_nvfuser")'),a2=s(" -  nvFuser with AotAutograd. "),br=l("a"),l2=s("Read more"),r2=p(),Mr=l("li"),Vh=l("code"),o2=s('dynamo.optimize("aot_cudagraphs")'),s2=s(" - cudagraphs with AotAutograd. "),vr=l("a"),i2=s("Read more"),Nc=p(),gr=l("p"),Hh=l("strong"),n2=s("Inference-only backend"),h2=s("s:"),Qc=p(),P=l("ul"),Tr=l("li"),Oh=l("code"),p2=s('dynamo.optimize("ofi")'),f2=s(" -  Uses Torchscript optimize_for_inference.  "),Ur=l("a"),m2=s("Read more"),c2=p(),_r=l("li"),qh=l("code"),d2=s('dynamo.optimize("fx2trt")'),u2=s(" -  Uses Nvidia TensorRT for inference optimizations.  "),Jr=l("a"),w2=s("Read more"),y2=p(),Er=l("li"),Kh=l("code"),b2=s('dynamo.optimize("onnxrt")'),M2=s(" -  Uses ONNXRT for inference on CPU/GPU.  "),Ar=l("a"),v2=s("Read more"),g2=p(),jr=l("li"),ep=l("code"),T2=s('dynamo.optimize("ipex")'),U2=s(" -  Uses IPEX for inference on CPU.  "),Ir=l("a"),_2=s("Read more"),this.h()},l(e){const n=q3('[data-svelte="svelte-1phssyn"]',document.head);g=r(n,"META",{name:!0,content:!0}),n.forEach(t),E=f(e),U=r(e,"H1",{class:!0});var kr=o(U);$=r(kr,"A",{id:!0,class:!0,href:!0});var tp=o($);Mi=r(tp,"SPAN",{});var B2=o(Mi);d(Ot.$$.fragment,B2),B2.forEach(t),tp.forEach(t),uu=f(kr),vi=r(kr,"SPAN",{});var N2=o(vi);wu=i(N2,"Efficient Training on a Single GPU"),N2.forEach(t),kr.forEach(t),rp=f(e),Be=r(e,"P",{});var Fc=o(Be);yu=i(Fc,"This guide focuses on training large models efficiently on a single GPU. These approaches are still valid if you have access to a machine with multiple GPUs but you will also have access to additional methods outlined in the "),Cr=r(Fc,"A",{href:!0});var Q2=o(Cr);bu=i(Q2,"multi-GPU section"),Q2.forEach(t),Mu=i(Fc,"."),Fc.forEach(t),op=f(e),N=r(e,"P",{});var Vs=o(N);vu=i(Vs,"In this section we have a look at a few tricks to reduce the memory footprint and speed up training for large models and how they are integrated in the "),Gr=r(Vs,"A",{href:!0});var Z2=o(Gr);gu=i(Z2,"Trainer"),Z2.forEach(t),Tu=i(Vs," and "),qt=r(Vs,"A",{href:!0,rel:!0});var F2=o(qt);Uu=i(F2,"\u{1F917} Accelerate"),F2.forEach(t),_u=i(Vs,". Each method can improve speed or memory usage which is summarized in the table below:"),Vs.forEach(t),sp=f(e),Ne=r(e,"TABLE",{});var xc=o(Ne);gi=r(xc,"THEAD",{});var x2=o(gi);le=r(x2,"TR",{});var Hs=o(le);Sr=r(Hs,"TH",{align:!0});var Y2=o(Sr);Ju=i(Y2,"Method"),Y2.forEach(t),Eu=f(Hs),Rr=r(Hs,"TH",{align:!0});var L2=o(Rr);Au=i(L2,"Speed"),L2.forEach(t),ju=f(Hs),Pr=r(Hs,"TH",{align:!0});var D2=o(Pr);Iu=i(D2,"Memory"),D2.forEach(t),Hs.forEach(t),x2.forEach(t),ku=f(xc),_=r(xc,"TBODY",{});var I=o(_);re=r(I,"TR",{});var Os=o(re);Wr=r(Os,"TD",{align:!0});var X2=o(Wr);$u=i(X2,"Gradient accumulation"),X2.forEach(t),zu=f(Os),Br=r(Os,"TD",{align:!0});var V2=o(Br);Cu=i(V2,"No"),V2.forEach(t),Gu=f(Os),Nr=r(Os,"TD",{align:!0});var H2=o(Nr);Su=i(H2,"Yes"),H2.forEach(t),Os.forEach(t),Ru=f(I),oe=r(I,"TR",{});var qs=o(oe);Qr=r(qs,"TD",{align:!0});var O2=o(Qr);Pu=i(O2,"Gradient checkpointing"),O2.forEach(t),Wu=f(qs),Zr=r(qs,"TD",{align:!0});var q2=o(Zr);Bu=i(q2,"No"),q2.forEach(t),Nu=f(qs),Fr=r(qs,"TD",{align:!0});var K2=o(Fr);Qu=i(K2,"Yes"),K2.forEach(t),qs.forEach(t),Zu=f(I),se=r(I,"TR",{});var Ks=o(se);xr=r(Ks,"TD",{align:!0});var eU=o(xr);Fu=i(eU,"Mixed precision training"),eU.forEach(t),xu=f(Ks),Yr=r(Ks,"TD",{align:!0});var tU=o(Yr);Yu=i(tU,"Yes"),tU.forEach(t),Lu=f(Ks),Lr=r(Ks,"TD",{align:!0});var aU=o(Lr);Du=i(aU,"(No)"),aU.forEach(t),Ks.forEach(t),Xu=f(I),ie=r(I,"TR",{});var ei=o(ie);Dr=r(ei,"TD",{align:!0});var lU=o(Dr);Vu=i(lU,"Batch size"),lU.forEach(t),Hu=f(ei),Xr=r(ei,"TD",{align:!0});var rU=o(Xr);Ou=i(rU,"Yes"),rU.forEach(t),qu=f(ei),Vr=r(ei,"TD",{align:!0});var oU=o(Vr);Ku=i(oU,"Yes"),oU.forEach(t),ei.forEach(t),ew=f(I),ne=r(I,"TR",{});var ti=o(ne);Hr=r(ti,"TD",{align:!0});var sU=o(Hr);tw=i(sU,"Optimizer choice"),sU.forEach(t),aw=f(ti),Or=r(ti,"TD",{align:!0});var iU=o(Or);lw=i(iU,"Yes"),iU.forEach(t),rw=f(ti),qr=r(ti,"TD",{align:!0});var nU=o(qr);ow=i(nU,"Yes"),nU.forEach(t),ti.forEach(t),sw=f(I),he=r(I,"TR",{});var ai=o(he);Kr=r(ai,"TD",{align:!0});var hU=o(Kr);iw=i(hU,"DataLoader"),hU.forEach(t),nw=f(ai),eo=r(ai,"TD",{align:!0});var pU=o(eo);hw=i(pU,"Yes"),pU.forEach(t),pw=f(ai),to=r(ai,"TD",{align:!0});var fU=o(to);fw=i(fU,"No"),fU.forEach(t),ai.forEach(t),mw=f(I),pe=r(I,"TR",{});var li=o(pe);ao=r(li,"TD",{align:!0});var mU=o(ao);cw=i(mU,"DeepSpeed Zero"),mU.forEach(t),dw=f(li),lo=r(li,"TD",{align:!0});var cU=o(lo);uw=i(cU,"No"),cU.forEach(t),ww=f(li),ro=r(li,"TD",{align:!0});var dU=o(ro);yw=i(dU,"Yes"),dU.forEach(t),li.forEach(t),I.forEach(t),xc.forEach(t),ip=f(e),oo=r(e,"P",{});var uU=o(oo);bw=i(uU,"A bracket means that it might not be strictly the case but is usually either not a main concern or negligible. Before we start make sure you have installed the following libraries:"),uU.forEach(t),np=f(e),d(Kt.$$.fragment,e),hp=f(e),Q=r(e,"P",{});var ri=o(Q);Mw=i(ri,"The "),Ti=r(ri,"CODE",{});var wU=o(Ti);vw=i(wU,"nvidia-ml-py3"),wU.forEach(t),gw=i(ri," library allows us to monitor the memory usage of the models from within Python. You might be familiar with the "),Ui=r(ri,"CODE",{});var yU=o(Ui);Tw=i(yU,"nvidia-smi"),yU.forEach(t),Uw=i(ri," command in the terminal - this library allows to access the same information in Python directly."),ri.forEach(t),pp=f(e),Qe=r(e,"P",{});var Yc=o(Qe);_w=i(Yc,"Then we create some dummy data. We create random token IDs between 100 and 30000 and binary labels for a classifier. In total we get 512 sequences each with length 512 and store them in a "),ea=r(Yc,"A",{href:!0,rel:!0});var bU=o(ea);Jw=i(bU,"Dataset"),bU.forEach(t),Ew=i(Yc," with PyTorch format."),Yc.forEach(t),fp=f(e),d(ta.$$.fragment,e),mp=f(e),Ze=r(e,"P",{});var Lc=o(Ze);Aw=i(Lc,"We want to print some summary statistics for the GPU utilization and the training run with the "),so=r(Lc,"A",{href:!0});var MU=o(so);jw=i(MU,"Trainer"),MU.forEach(t),Iw=i(Lc,". We setup a two helper functions to do just that:"),Lc.forEach(t),cp=f(e),d(aa.$$.fragment,e),dp=f(e),io=r(e,"P",{});var vU=o(io);kw=i(vU,"Let\u2019s verify that we start with a free GPU memory:"),vU.forEach(t),up=f(e),d(la.$$.fragment,e),wp=f(e),no=r(e,"P",{});var gU=o(no);$w=i(gU,"That looks good: the GPU memory is not occupied as we would expect before we load any models. If that\u2019s not the case on your machine make sure to stop all processes that are using GPU memory. However, not all free GPU memory can be used by the user. When a model is loaded to the GPU also the kernels are loaded which can take up 1-2GB of memory. To see how much it is we load a tiny tensor into the GPU which triggers the kernels to be loaded as well."),gU.forEach(t),yp=f(e),d(ra.$$.fragment,e),bp=f(e),ho=r(e,"P",{});var TU=o(ho);zw=i(TU,"We see that the kernels alone take up 1.3GB of GPU memory. Now let\u2019s see how much space the model uses."),TU.forEach(t),Mp=f(e),fe=r(e,"H2",{class:!0});var Dc=o(fe);Fe=r(Dc,"A",{id:!0,class:!0,href:!0});var UU=o(Fe);_i=r(UU,"SPAN",{});var _U=o(_i);d(oa.$$.fragment,_U),_U.forEach(t),UU.forEach(t),Cw=f(Dc),Ji=r(Dc,"SPAN",{});var JU=o(Ji);Gw=i(JU,"Load Model"),JU.forEach(t),Dc.forEach(t),vp=f(e),xe=r(e,"P",{});var Xc=o(xe);Sw=i(Xc,"First, we load the "),Ei=r(Xc,"CODE",{});var EU=o(Ei);Rw=i(EU,"bert-large-uncased"),EU.forEach(t),Pw=i(Xc," model. We load the model weights directly to the GPU so that we can check how much space just weights use."),Xc.forEach(t),gp=f(e),d(sa.$$.fragment,e),Tp=f(e),Ye=r(e,"P",{});var Vc=o(Ye);Ww=i(Vc,"We can see that the model weights alone take up 1.3 GB of the GPU memory. The exact number depends on the specific GPU you are using. Note that on newer GPUs a model can sometimes take up more space since the weights are loaded in an optimized fashion that speeds up the usage of the model. Now we can also quickly check if we get the same result as with "),Ai=r(Vc,"CODE",{});var AU=o(Ai);Bw=i(AU,"nvidia-smi"),AU.forEach(t),Nw=i(Vc," CLI:"),Vc.forEach(t),Up=f(e),d(ia.$$.fragment,e),_p=f(e),d(na.$$.fragment,e),Jp=f(e),po=r(e,"P",{});var jU=o(po);Qw=i(jU,"We get the same number as before and you can also see that we are using a V100 GPU with 16GB of memory. So now we can start training the model and see how the GPU memory consumption changes. First, we set up a few standard training arguments that we will use across all our experiments:"),jU.forEach(t),Ep=f(e),d(ha.$$.fragment,e),Ap=f(e),d(Le.$$.fragment,e),jp=f(e),me=r(e,"H2",{class:!0});var Hc=o(me);De=r(Hc,"A",{id:!0,class:!0,href:!0});var IU=o(De);ji=r(IU,"SPAN",{});var kU=o(ji);d(pa.$$.fragment,kU),kU.forEach(t),IU.forEach(t),Zw=f(Hc),Ii=r(Hc,"SPAN",{});var $U=o(Ii);Fw=i($U,"Vanilla Training"),$U.forEach(t),Hc.forEach(t),Ip=f(e),Xe=r(e,"P",{});var Oc=o(Xe);xw=i(Oc,"As a first experiment we will use the "),fo=r(Oc,"A",{href:!0});var zU=o(fo);Yw=i(zU,"Trainer"),zU.forEach(t),Lw=i(Oc," and train the model without any further modifications and a batch size of 4:"),Oc.forEach(t),kp=f(e),d(fa.$$.fragment,e),$p=f(e),d(ma.$$.fragment,e),zp=f(e),mo=r(e,"P",{});var CU=o(mo);Dw=i(CU,"We see that already a relatively small batch size almost fills up our GPU\u2019s entire memory. However, a larger batch size can often result in faster model convergence or better end performance. So ideally we want to tune the batch size to our model\u2019s needs and not to the GPU limitations. What\u2019s interesting is that we use much more memory than the size of the model. To understand a bit better why this is the case let\u2019s have look at a model\u2019s operations and memory needs."),CU.forEach(t),Cp=f(e),ce=r(e,"H2",{class:!0});var qc=o(ce);Ve=r(qc,"A",{id:!0,class:!0,href:!0});var GU=o(Ve);ki=r(GU,"SPAN",{});var SU=o(ki);d(ca.$$.fragment,SU),SU.forEach(t),GU.forEach(t),Xw=f(qc),$i=r(qc,"SPAN",{});var RU=o($i);Vw=i(RU,"Anatomy of Model's Operations"),RU.forEach(t),qc.forEach(t),Gp=f(e),co=r(e,"P",{});var PU=o(co);Hw=i(PU,"Transformers architecture includes 3 main groups of operations grouped below by compute-intensity."),PU.forEach(t),Sp=f(e),Z=r(e,"OL",{});var oi=o(Z);da=r(oi,"LI",{});var Kc=o(da);zi=r(Kc,"P",{});var WU=o(zi);Ci=r(WU,"STRONG",{});var BU=o(Ci);Ow=i(BU,"Tensor Contractions"),BU.forEach(t),WU.forEach(t),qw=f(Kc),ua=r(Kc,"P",{});var ed=o(ua);Kw=i(ed,"Linear layers and components of Multi-Head Attention all do batched "),Gi=r(ed,"STRONG",{});var NU=o(Gi);ey=i(NU,"matrix-matrix multiplications"),NU.forEach(t),ty=i(ed,". These operations are the most compute-intensive part of training a transformer."),ed.forEach(t),Kc.forEach(t),ay=f(oi),wa=r(oi,"LI",{});var td=o(wa);Si=r(td,"P",{});var QU=o(Si);Ri=r(QU,"STRONG",{});var ZU=o(Ri);ly=i(ZU,"Statistical Normalizations"),ZU.forEach(t),QU.forEach(t),ry=f(td),ya=r(td,"P",{});var ad=o(ya);oy=i(ad,"Softmax and layer normalization are less compute-intensive than tensor contractions, and involve one or more "),Pi=r(ad,"STRONG",{});var FU=o(Pi);sy=i(FU,"reduction operations"),FU.forEach(t),iy=i(ad,", the result of which is then applied via a map."),ad.forEach(t),td.forEach(t),ny=f(oi),ba=r(oi,"LI",{});var ld=o(ba);Wi=r(ld,"P",{});var xU=o(Wi);Bi=r(xU,"STRONG",{});var YU=o(Bi);hy=i(YU,"Element-wise Operators"),YU.forEach(t),xU.forEach(t),py=f(ld),Ma=r(ld,"P",{});var rd=o(Ma);fy=i(rd,"These are the remaining operators: "),Ni=r(rd,"STRONG",{});var LU=o(Ni);my=i(LU,"biases, dropout, activations, and residual connections"),LU.forEach(t),cy=i(rd,". These are the least compute-intensive operations."),rd.forEach(t),ld.forEach(t),oi.forEach(t),Rp=f(e),uo=r(e,"P",{});var DU=o(uo);dy=i(DU,"This knowledge can be helpful to know when analyzing performance bottlenecks."),DU.forEach(t),Pp=f(e),va=r(e,"P",{});var J2=o(va);uy=i(J2,"This summary is derived from "),ga=r(J2,"A",{href:!0,rel:!0});var XU=o(ga);wy=i(XU,"Data Movement Is All You Need: A Case Study on Optimizing Transformers 2020"),XU.forEach(t),J2.forEach(t),Wp=f(e),de=r(e,"H2",{class:!0});var od=o(de);He=r(od,"A",{id:!0,class:!0,href:!0});var VU=o(He);Qi=r(VU,"SPAN",{});var HU=o(Qi);d(Ta.$$.fragment,HU),HU.forEach(t),VU.forEach(t),yy=f(od),Zi=r(od,"SPAN",{});var OU=o(Zi);by=i(OU,"Anatomy of Model's Memory"),OU.forEach(t),od.forEach(t),Bp=i(e,`

We've seen that training the model uses much more memory than just putting the model on the GPU. This is because there are many components during training that use GPU memory. The components on GPU memory are the following:
1. model weights
2. optimizer states
3. gradients
4. forward activations saved for gradient computation
5. temporary buffers
6. functionality-specific memory
`),wo=r(e,"P",{});var qU=o(wo);My=i(qU,"A typical model trained in mixed precision with AdamW requires 18 bytes per model parameter plus activation memory. For inference there are no optimizer states and gradients, so we can subtract those. And thus we end up with 6 bytes per model parameter for mixed precision inference, plus activation memory."),qU.forEach(t),Np=f(e),yo=r(e,"P",{});var KU=o(yo);vy=i(KU,"Let\u2019s look at the details."),KU.forEach(t),Qp=f(e),bo=r(e,"P",{});var e0=o(bo);Fi=r(e0,"STRONG",{});var t0=o(Fi);gy=i(t0,"Model Weights:"),t0.forEach(t),e0.forEach(t),Zp=f(e),Oe=r(e,"UL",{});var sd=o(Oe);xi=r(sd,"LI",{});var a0=o(xi);Ty=i(a0,"4 bytes * number of parameters for fp32 training"),a0.forEach(t),Uy=f(sd),Yi=r(sd,"LI",{});var l0=o(Yi);_y=i(l0,"6 bytes * number of parameters for mixed precision training (maintains a model in fp32 and one in fp16 in memory)"),l0.forEach(t),sd.forEach(t),Fp=f(e),Mo=r(e,"P",{});var r0=o(Mo);Li=r(r0,"STRONG",{});var o0=o(Li);Jy=i(o0,"Optimizer States:"),o0.forEach(t),r0.forEach(t),xp=f(e),F=r(e,"UL",{});var si=o(F);Di=r(si,"LI",{});var s0=o(Di);Ey=i(s0,"8 bytes * number of parameters for normal AdamW (maintains 2 states)"),s0.forEach(t),Ay=f(si),vo=r(si,"LI",{});var E2=o(vo);jy=i(E2,"2 bytes * number of parameters for 8-bit AdamW optimizers like "),Ua=r(E2,"A",{href:!0,rel:!0});var i0=o(Ua);Iy=i(i0,"bitsandbytes"),i0.forEach(t),E2.forEach(t),ky=f(si),Xi=r(si,"LI",{});var n0=o(Xi);$y=i(n0,"4 bytes * number of parameters for optimizers like SGD with momentum (maintains only 1 state)"),n0.forEach(t),si.forEach(t),Yp=f(e),go=r(e,"P",{});var h0=o(go);Vi=r(h0,"STRONG",{});var p0=o(Vi);zy=i(p0,"Gradients"),p0.forEach(t),h0.forEach(t),Lp=f(e),To=r(e,"UL",{});var f0=o(To);Hi=r(f0,"LI",{});var m0=o(Hi);Cy=i(m0,"4 bytes * number of parameters for either fp32 or mixed precision training (gradients are always kept in fp32)"),m0.forEach(t),f0.forEach(t),Dp=f(e),Uo=r(e,"P",{});var c0=o(Uo);Oi=r(c0,"STRONG",{});var d0=o(Oi);Gy=i(d0,"Forward Activations"),d0.forEach(t),c0.forEach(t),Xp=f(e),_o=r(e,"UL",{});var u0=o(_o);qi=r(u0,"LI",{});var w0=o(qi);Sy=i(w0,"size depends on many factors, the key ones being sequence length, hidden size and batch size."),w0.forEach(t),u0.forEach(t),Vp=f(e),Jo=r(e,"P",{});var y0=o(Jo);Ry=i(y0,"There are the input and output that are being passed and returned by the forward and the backward functions and the forward activations saved for gradient computation."),y0.forEach(t),Hp=f(e),Eo=r(e,"P",{});var b0=o(Eo);Ki=r(b0,"STRONG",{});var M0=o(Ki);Py=i(M0,"Temporary Memory"),M0.forEach(t),b0.forEach(t),Op=f(e),Ao=r(e,"P",{});var v0=o(Ao);Wy=i(v0,"Additionally there are all kinds of temporary variables which get released once the calculation is done, but in the moment these could require additional memory and could push to OOM. Therefore when coding it\u2019s crucial to think strategically about such temporary variables and sometimes to explicitly free those as soon as they are no longer needed."),v0.forEach(t),qp=f(e),jo=r(e,"P",{});var g0=o(jo);en=r(g0,"STRONG",{});var T0=o(en);By=i(T0,"Functionality-specific memory"),T0.forEach(t),g0.forEach(t),Kp=f(e),Io=r(e,"P",{});var U0=o(Io);Ny=i(U0,"Then your software could have special memory needs. For example, when generating text using beam search, the software needs to maintain multiple copies of inputs and outputs."),U0.forEach(t),ef=f(e),ko=r(e,"P",{});var _0=o(ko);qe=r(_0,"STRONG",{});var ap=o(qe);tn=r(ap,"CODE",{});var J0=o(tn);Qy=i(J0,"forward"),J0.forEach(t),Zy=i(ap," vs "),an=r(ap,"CODE",{});var E0=o(an);Fy=i(E0,"backward"),E0.forEach(t),xy=i(ap," Execution Speed"),ap.forEach(t),_0.forEach(t),tf=f(e),$o=r(e,"P",{});var A0=o($o);Yy=i(A0,"For convolutions and linear layers there are 2x flops in the backward compared to the forward, which generally translates into ~2x slower (sometimes more, because sizes in the backward tend to be more awkward). Activations are usually bandwidth-limited, and it\u2019s typical for an activation to have to read more data in the backward than in the forward (e.g. activation forward reads once, writes once, activation backward reads twice, gradOutput and output of the forward, and writes once, gradInput)."),A0.forEach(t),af=f(e),zo=r(e,"P",{});var j0=o(zo);Ly=i(j0,"So there are potentially a few places where we could save GPU memory or speed up operations. Let\u2019s start with a simple optimization: choosing the right batch size."),j0.forEach(t),lf=f(e),ue=r(e,"H2",{class:!0});var id=o(ue);Ke=r(id,"A",{id:!0,class:!0,href:!0});var I0=o(Ke);ln=r(I0,"SPAN",{});var k0=o(ln);d(_a.$$.fragment,k0),k0.forEach(t),I0.forEach(t),Dy=f(id),rn=r(id,"SPAN",{});var $0=o(rn);Xy=i($0,"Batch sizes"),$0.forEach(t),id.forEach(t),rf=f(e),Co=r(e,"P",{});var z0=o(Co);Vy=i(z0,"One gets the most efficient performance when batch sizes and input/output neuron counts are divisible by a certain number, which typically starts at 8, but can be much higher as well. That number varies a lot depending on the specific hardware being used and the dtype of the model."),z0.forEach(t),of=f(e),x=r(e,"P",{});var ii=o(x);Hy=i(ii,"For example for fully connected layers (which correspond to GEMMs), NVIDIA provides recommendations for "),Ja=r(ii,"A",{href:!0,rel:!0});var C0=o(Ja);Oy=i(C0,"input/output neuron counts"),C0.forEach(t),qy=i(ii," and "),Ea=r(ii,"A",{href:!0,rel:!0});var G0=o(Ea);Ky=i(G0,"batch size"),G0.forEach(t),eb=i(ii,"."),ii.forEach(t),sf=f(e),Aa=r(e,"P",{});var A2=o(Aa);ja=r(A2,"A",{href:!0,rel:!0});var S0=o(ja);tb=i(S0,"Tensor Core Requirements"),S0.forEach(t),ab=i(A2," define the multiplier based on the dtype and the hardware. For example, for fp16 a multiple of 8 is recommended, but on A100 it\u2019s 64!"),A2.forEach(t),nf=f(e),et=r(e,"P",{});var nd=o(et);lb=i(nd,"For parameters that are small, there is also "),Ia=r(nd,"A",{href:!0,rel:!0});var R0=o(Ia);rb=i(R0,"Dimension Quantization Effects"),R0.forEach(t),ob=i(nd," to consider, this is where tiling happens and the right multiplier can have a significant speedup."),nd.forEach(t),hf=f(e),we=r(e,"H2",{class:!0});var hd=o(we);tt=r(hd,"A",{id:!0,class:!0,href:!0});var P0=o(tt);on=r(P0,"SPAN",{});var W0=o(on);d(ka.$$.fragment,W0),W0.forEach(t),P0.forEach(t),sb=f(hd),sn=r(hd,"SPAN",{});var B0=o(sn);ib=i(B0,"Gradient Accumulation"),B0.forEach(t),hd.forEach(t),pf=f(e),Go=r(e,"P",{});var N0=o(Go);nb=i(N0,"The idea behind gradient accumulation is to instead of calculating the gradients for the whole batch at once to do it in smaller steps. The way we do that is to calculate the gradients iteratively in smaller batches by doing a forward and backward pass through the model and accumulating the gradients in the process. When enough gradients are accumulated we run the model\u2019s optimization step. This way we can easily increase the overall batch size to numbers that would never fit into the GPU\u2019s memory. In turn, however, the added forward and backward passes can slow down the training a bit."),N0.forEach(t),ff=f(e),z=r(e,"P",{});var Yt=o(z);hb=i(Yt,"We can use gradient accumulation in the "),So=r(Yt,"A",{href:!0});var Q0=o(So);pb=i(Q0,"Trainer"),Q0.forEach(t),fb=i(Yt," by simply adding the "),nn=r(Yt,"CODE",{});var Z0=o(nn);mb=i(Z0,"gradient_accumulation_steps"),Z0.forEach(t),cb=i(Yt," argument to "),Ro=r(Yt,"A",{href:!0});var F0=o(Ro);db=i(F0,"TrainingArguments"),F0.forEach(t),ub=i(Yt,". Let\u2019s see how it impacts the models memory footprint:"),Yt.forEach(t),mf=f(e),d($a.$$.fragment,e),cf=f(e),d(za.$$.fragment,e),df=f(e),A=r(e,"P",{});var te=o(A);wb=i(te,"We can see that the memory footprint was dramatically reduced at the cost of being only slightly slower than the vanilla run. Of course, this would change as you increase the number of accumulation steps. In general you would want to max out the GPU usage as much as possible. So in our case, the batch_size of 4 was already pretty close to the GPU\u2019s limit. If we wanted to train with a batch size of 64 we should not use "),hn=r(te,"CODE",{});var x0=o(hn);yb=i(x0,"per_device_train_batch_size=1"),x0.forEach(t),bb=i(te," and "),pn=r(te,"CODE",{});var Y0=o(pn);Mb=i(Y0,"gradient_accumulation_steps=64"),Y0.forEach(t),vb=i(te," but instead "),fn=r(te,"CODE",{});var L0=o(fn);gb=i(L0,"per_device_train_batch_size=4"),L0.forEach(t),Tb=i(te," and "),mn=r(te,"CODE",{});var D0=o(mn);Ub=i(D0,"gradient_accumulation_steps=16"),D0.forEach(t),_b=i(te," which has the same effective batch size while making better use of the available GPU resources."),te.forEach(t),uf=f(e),Y=r(e,"P",{});var ni=o(Y);Jb=i(ni,"For more details see the benchmarks for "),Ca=r(ni,"A",{href:!0,rel:!0});var X0=o(Ca);Eb=i(X0,"RTX-3090"),X0.forEach(t),Ab=i(ni,`
and `),Ga=r(ni,"A",{href:!0,rel:!0});var V0=o(Ga);jb=i(V0,"A100"),V0.forEach(t),Ib=i(ni,"."),ni.forEach(t),wf=f(e),Po=r(e,"P",{});var H0=o(Po);kb=i(H0,"Next we have a look at another trick to save a little bit more GPU memory called gradient checkpointing."),H0.forEach(t),yf=f(e),ye=r(e,"H2",{class:!0});var pd=o(ye);at=r(pd,"A",{id:!0,class:!0,href:!0});var O0=o(at);cn=r(O0,"SPAN",{});var q0=o(cn);d(Sa.$$.fragment,q0),q0.forEach(t),O0.forEach(t),$b=f(pd),dn=r(pd,"SPAN",{});var K0=o(dn);zb=i(K0,"Gradient Checkpointing"),K0.forEach(t),pd.forEach(t),bf=f(e),Wo=r(e,"P",{});var e_=o(Wo);Cb=i(e_,"Even when we set the batch size to 1 and use gradient accumulation we can still run out of memory when working with large models. In order to compute the gradients during the backward pass all activations from the forward pass are normally saved. This can create a big memory overhead. Alternatively, one could forget all activations during the forward pass and recompute them on demand during the backward pass. This would however add a significant computational overhead and slow down training."),e_.forEach(t),Mf=f(e),lt=r(e,"P",{});var fd=o(lt);Gb=i(fd,"Gradient checkpointing strikes a compromise between the two approaches and saves strategically selected activations throughout the computational graph so only a fraction of the activations need to be re-computed for the gradients. See "),Ra=r(fd,"A",{href:!0,rel:!0});var t_=o(Ra);Sb=i(t_,"this great article"),t_.forEach(t),Rb=i(fd," explaining the ideas behind gradient checkpointing."),fd.forEach(t),vf=f(e),L=r(e,"P",{});var hi=o(L);Pb=i(hi,"To enable gradient checkpointing in the "),Bo=r(hi,"A",{href:!0});var a_=o(Bo);Wb=i(a_,"Trainer"),a_.forEach(t),Bb=i(hi," we only need to pass it as a flag to the "),No=r(hi,"A",{href:!0});var l_=o(No);Nb=i(l_,"TrainingArguments"),l_.forEach(t),Qb=i(hi,". Everything else is handled under the hood:"),hi.forEach(t),gf=f(e),d(Pa.$$.fragment,e),Tf=f(e),d(Wa.$$.fragment,e),Uf=f(e),Qo=r(e,"P",{});var r_=o(Qo);Zb=i(r_,"We can see that this saved some more memory but at the same time training became a bit slower. A general rule of thumb is that gradient checkpointing slows down training by about 20%. Let\u2019s have a look at another method with which we can regain some speed: mixed precision training."),r_.forEach(t),_f=f(e),be=r(e,"H2",{class:!0});var md=o(be);rt=r(md,"A",{id:!0,class:!0,href:!0});var o_=o(rt);un=r(o_,"SPAN",{});var s_=o(un);d(Ba.$$.fragment,s_),s_.forEach(t),o_.forEach(t),Fb=f(md),wn=r(md,"SPAN",{});var i_=o(wn);xb=i(i_,"Floating Data Types"),i_.forEach(t),md.forEach(t),Jf=f(e),Zo=r(e,"P",{});var n_=o(Zo);Yb=i(n_,"The idea of mixed precision training is that not all variables need to be stored in full (32-bit) floating point precision. If we can reduce the precision the variables and their computations are faster. Here are the commonly used floating point data types choice of which impacts both memory usage and throughput:"),n_.forEach(t),Ef=f(e),C=r(e,"UL",{});var Lt=o(C);Na=r(Lt,"LI",{});var cd=o(Na);Lb=i(cd,"fp32 ("),yn=r(cd,"CODE",{});var h_=o(yn);Db=i(h_,"float32"),h_.forEach(t),Xb=i(cd,")"),cd.forEach(t),Vb=f(Lt),Qa=r(Lt,"LI",{});var dd=o(Qa);Hb=i(dd,"fp16 ("),bn=r(dd,"CODE",{});var p_=o(bn);Ob=i(p_,"float16"),p_.forEach(t),qb=i(dd,")"),dd.forEach(t),Kb=f(Lt),Za=r(Lt,"LI",{});var ud=o(Za);eM=i(ud,"bf16 ("),Mn=r(ud,"CODE",{});var f_=o(Mn);tM=i(f_,"bfloat16"),f_.forEach(t),aM=i(ud,")"),ud.forEach(t),lM=f(Lt),vn=r(Lt,"LI",{});var m_=o(vn);rM=i(m_,"tf32 (CUDA internal data type)"),m_.forEach(t),Lt.forEach(t),Af=f(e),Fo=r(e,"P",{});var c_=o(Fo);oM=i(c_,"Here is a diagram that shows how these data types correlate to each other."),c_.forEach(t),jf=f(e),Me=r(e,"P",{});var lp=o(Me);xo=r(lp,"IMG",{src:!0,alt:!0}),sM=i(lp,`
(source: `),Fa=r(lp,"A",{href:!0,rel:!0});var d_=o(Fa);iM=i(d_,"NVIDIA Blog"),d_.forEach(t),nM=i(lp,")"),lp.forEach(t),If=f(e),Yo=r(e,"P",{});var u_=o(Yo);hM=i(u_,"While fp16 and fp32 have been around for quite some time, bf16 and tf32 are only available on the Ampere architecture GPUS and TPUs support bf16 as well. Let\u2019s start with the most commonly used method which is FP16 training/"),u_.forEach(t),kf=f(e),ve=r(e,"H3",{class:!0});var wd=o(ve);ot=r(wd,"A",{id:!0,class:!0,href:!0});var w_=o(ot);gn=r(w_,"SPAN",{});var y_=o(gn);d(xa.$$.fragment,y_),y_.forEach(t),w_.forEach(t),pM=f(wd),Tn=r(wd,"SPAN",{});var b_=o(Tn);fM=i(b_,"FP16 Training"),b_.forEach(t),wd.forEach(t),$f=f(e),D=r(e,"P",{});var pi=o(D);mM=i(pi,"The idea of mixed precision training is that not all variables need to be stored in full (32-bit) floating point precision. If we can reduce the precision the variales and their computations are faster. The main advantage comes from saving the activations in half (16-bit) precision. Although the gradients are also computed in half precision they are converted back to full precision for the optimization step so no memory is saved here. Since the model is present on the GPU in both 16-bit and 32-bit precision this can use more GPU memory (1.5x the original model is on the GPU), especially for small batch sizes. Since some computations are performed in full and some in half precision this approach is also called mixed precision training. Enabling mixed precision training is also just a matter of setting the "),Un=r(pi,"CODE",{});var M_=o(Un);cM=i(M_,"fp16"),M_.forEach(t),dM=i(pi," flag to "),_n=r(pi,"CODE",{});var v_=o(_n);uM=i(v_,"True"),v_.forEach(t),wM=i(pi,":"),pi.forEach(t),zf=f(e),d(Ya.$$.fragment,e),Cf=f(e),d(La.$$.fragment,e),Gf=f(e),Lo=r(e,"P",{});var g_=o(Lo);yM=i(g_,"We can see that this is almost twice as fast as the vanilla training. Let\u2019s add it to the mix of the previous methods:"),g_.forEach(t),Sf=f(e),d(Da.$$.fragment,e),Rf=f(e),d(Xa.$$.fragment,e),Pf=f(e),Do=r(e,"P",{});var T_=o(Do);bM=i(T_,"We can see that with these tweaks we use about half the GPU memory as at the beginning while also being slightly faster."),T_.forEach(t),Wf=f(e),ge=r(e,"H3",{class:!0});var yd=o(ge);st=r(yd,"A",{id:!0,class:!0,href:!0});var U_=o(st);Jn=r(U_,"SPAN",{});var __=o(Jn);d(Va.$$.fragment,__),__.forEach(t),U_.forEach(t),MM=f(yd),En=r(yd,"SPAN",{});var J_=o(En);vM=i(J_,"BF16"),J_.forEach(t),yd.forEach(t),Bf=i(e,"\n\nIf you have access to a Ampere or newer hardware you can use bf16 for your training and evaluation. While bf16 has a worse precision than fp16, it has a much much bigger dynamic range. Therefore, if in the past you were experiencing overflow issues while training the model, bf16 will prevent this from happening most of the time. Remember that in fp16 the biggest number you can have is `65535` and any number above that will overflow. A bf16 number can be as large as `3.39e+38` (!) which is about the same as fp32 - because both have 8-bits used for the numerical range.\n"),Xo=r(e,"P",{});var E_=o(Xo);gM=i(E_,"You can enable BF16 in the \u{1F917} Trainer with:"),E_.forEach(t),Nf=f(e),d(Ha.$$.fragment,e),Qf=f(e),Te=r(e,"H3",{class:!0});var bd=o(Te);it=r(bd,"A",{id:!0,class:!0,href:!0});var A_=o(it);An=r(A_,"SPAN",{});var j_=o(An);d(Oa.$$.fragment,j_),j_.forEach(t),A_.forEach(t),TM=f(bd),jn=r(bd,"SPAN",{});var I_=o(jn);UM=i(I_,"TF32"),I_.forEach(t),bd.forEach(t),Zf=i(e,`

The Ampere hardware uses a magical data type called tf32. It has the same numerical range as fp32 (8-bits), but instead of 23 bits precision it has only 10 bits (same as fp16) and uses only 19 bits in total.
`),Vo=r(e,"P",{});var k_=o(Vo);_M=i(k_,"It\u2019s magical in the sense that you can use the normal fp32 training and/or inference code and by enabling tf32 support you can get up to 3x throughput improvement. All you need to do is to add this to your code:"),k_.forEach(t),Ff=f(e),d(qa.$$.fragment,e),xf=f(e),Ho=r(e,"P",{});var $_=o(Ho);JM=i($_,"When this is done CUDA will automatically switch to using tf32 instead of fp32 where it\u2019s possible. This, of course, assumes that the used GPU is from the Ampere series."),$_.forEach(t),Yf=f(e),nt=r(e,"P",{});var Md=o(nt);EM=i(Md,"Like all cases with reduced precision this may or may not be satisfactory for your needs, so you have to experiment and see. According to "),Ka=r(Md,"A",{href:!0,rel:!0});var z_=o(Ka);AM=i(z_,"NVIDIA research"),z_.forEach(t),jM=i(Md," the majority of machine learning training shouldn\u2019t be impacted and showed the same perplexity and convergence as the fp32 training."),Md.forEach(t),Lf=f(e),Oo=r(e,"P",{});var C_=o(Oo);IM=i(C_,"If you\u2019re already using fp16 or bf16 mixed precision it may help with the throughput as well."),C_.forEach(t),Df=f(e),qo=r(e,"P",{});var G_=o(qo);kM=i(G_,"You can enable this mode in the \u{1F917} Trainer with:"),G_.forEach(t),Xf=f(e),d(el.$$.fragment,e),Vf=f(e),Ko=r(e,"P",{});var S_=o(Ko);$M=i(S_,"By default the PyTorch default is used."),S_.forEach(t),Hf=f(e),X=r(e,"P",{});var fi=o(X);zM=i(fi,"Note: tf32 mode is internal to CUDA and can\u2019t be accessed directly via "),In=r(fi,"CODE",{});var R_=o(In);CM=i(R_,"tensor.to(dtype=torch.tf32)"),R_.forEach(t),GM=i(fi," as "),kn=r(fi,"CODE",{});var P_=o(kn);SM=i(P_,"torch.tf32"),P_.forEach(t),RM=i(fi," doesn\u2019t exist."),fi.forEach(t),Of=f(e),ht=r(e,"P",{});var vd=o(ht);PM=i(vd,"Note: you need "),$n=r(vd,"CODE",{});var W_=o($n);WM=i(W_,"torch>=1.7"),W_.forEach(t),BM=i(vd," to enjoy this feature."),vd.forEach(t),qf=f(e),V=r(e,"P",{});var mi=o(V);NM=i(mi,`You can also see a variety of benchmarks on tf32 vs other precisions:
`),tl=r(mi,"A",{href:!0,rel:!0});var B_=o(tl);QM=i(B_,"RTX-3090"),B_.forEach(t),ZM=i(mi,` and
`),al=r(mi,"A",{href:!0,rel:!0});var N_=o(al);FM=i(N_,"A100"),N_.forEach(t),xM=i(mi,"."),mi.forEach(t),Kf=f(e),es=r(e,"P",{});var Q_=o(es);YM=i(Q_,"We\u2019ve now seen how we can change the floating types to increase throughput, but we are not done, yet! There is another area where we can save GPU memory: the optimizer."),Q_.forEach(t),em=f(e),Ue=r(e,"H2",{class:!0});var gd=o(Ue);pt=r(gd,"A",{id:!0,class:!0,href:!0});var Z_=o(pt);zn=r(Z_,"SPAN",{});var F_=o(zn);d(ll.$$.fragment,F_),F_.forEach(t),Z_.forEach(t),LM=f(gd),Cn=r(gd,"SPAN",{});var x_=o(Cn);DM=i(x_,"Optimizer"),x_.forEach(t),gd.forEach(t),tm=f(e),ts=r(e,"P",{});var Y_=o(ts);XM=i(Y_,"The most common optimizer used to train transformer model is Adam or AdamW (Adam with weight decay). Adam achieves good convergence by storing the rolling average of the previous gradients which, however, adds an additional memory footprint of the order of the number of model parameters. One remedy to this is to use an alternative optimizer such as Adafactor, which works well for some models but often it has instability issues."),Y_.forEach(t),am=f(e),ft=r(e,"P",{});var Td=o(ft);VM=i(Td,"HF Trainer integrates a variety of optimisers that can be used out of box. To activate the desired optimizer simply pass the "),Gn=r(Td,"CODE",{});var L_=o(Gn);HM=i(L_,"--optim"),L_.forEach(t),OM=i(Td," flag to the command line."),Td.forEach(t),lm=f(e),as=r(e,"P",{});var D_=o(as);qM=i(D_,"To see which optimizers are currently supported:"),D_.forEach(t),rm=f(e),d(rl.$$.fragment,e),om=f(e),H=r(e,"P",{});var ci=o(H);KM=i(ci,"For example, if you have "),ol=r(ci,"A",{href:!0,rel:!0});var X_=o(ol);ev=i(X_,"NVIDIA/apex"),X_.forEach(t),tv=i(ci," installed "),Sn=r(ci,"CODE",{});var V_=o(Sn);av=i(V_,"--optim adamw_apex_fused"),V_.forEach(t),lv=i(ci," will give you the fastest training experience among all supported AdamW optimizers."),ci.forEach(t),sm=f(e),mt=r(e,"P",{});var Ud=o(mt);rv=i(Ud,"On the other hand "),sl=r(Ud,"A",{href:!0,rel:!0});var H_=o(sl);ov=i(H_,"8bit BNB optimizer"),H_.forEach(t),sv=i(Ud," can save 3/4 of memory normally used by a typical AdamW optimizer if it is configured to quantize all optimizer states, but in some situations only some optimizer states are quintized and then more memory is used."),Ud.forEach(t),im=f(e),ct=r(e,"P",{});var _d=o(ct);iv=i(_d,"Let\u2019s get a feel for the numbers and use for example use a 3B-parameter model, like "),Rn=r(_d,"CODE",{});var O_=o(Rn);nv=i(O_,"t5-3b"),O_.forEach(t),hv=i(_d,". Note that since a Gigabyte correpsonds to a billion bytes we can simply multiply the parameters (in billions) with the number of necessary bytes per parameter to get Gigabytes of GPU memory usage:"),_d.forEach(t),nm=f(e),O=r(e,"UL",{});var di=o(O);il=r(di,"LI",{});var Jd=o(il);pv=i(Jd,"A standard AdamW uses 8 bytes for each parameter, here the optimizer will need ("),Pn=r(Jd,"CODE",{});var q_=o(Pn);fv=i(q_,"8*3"),q_.forEach(t),mv=i(Jd,") 24GB of GPU memory."),Jd.forEach(t),cv=f(di),nl=r(di,"LI",{});var Ed=o(nl);dv=i(Ed,"Adafactor uses slightly more than 4 bytes, so ("),Wn=r(Ed,"CODE",{});var K_=o(Wn);uv=i(K_,"4*3"),K_.forEach(t),wv=i(Ed,") 12GB and then some extra."),Ed.forEach(t),yv=f(di),hl=r(di,"LI",{});var Ad=o(hl);bv=i(Ad,"8bit BNB quantized optimizer will use only ("),Bn=r(Ad,"CODE",{});var eJ=o(Bn);Mv=i(eJ,"2*3"),eJ.forEach(t),vv=i(Ad,") 6GB if all optimizer states are quantized."),Ad.forEach(t),di.forEach(t),hm=f(e),ls=r(e,"P",{});var tJ=o(ls);gv=i(tJ,"Let\u2019s have a look at Adafactor first."),tJ.forEach(t),pm=f(e),_e=r(e,"H3",{class:!0});var jd=o(_e);dt=r(jd,"A",{id:!0,class:!0,href:!0});var aJ=o(dt);Nn=r(aJ,"SPAN",{});var lJ=o(Nn);d(pl.$$.fragment,lJ),lJ.forEach(t),aJ.forEach(t),Tv=f(jd),Qn=r(jd,"SPAN",{});var rJ=o(Qn);Uv=i(rJ,"Adafactor"),rJ.forEach(t),jd.forEach(t),fm=f(e),ut=r(e,"P",{});var Id=o(ut);_v=i(Id,"Instead of keeping the rolling average for each element in the weight matrices Adafactor only stores aggregated information (row- and column-wise sums of the rolling averages) which reduces the footprint considerably. One downside of Adafactor is that in some instances convergence can be slower than Adam\u2019s so some experimentation is advised here. We can use Adafactor simply by setting "),Zn=r(Id,"CODE",{});var oJ=o(Zn);Jv=i(oJ,'optim="adafactor"'),oJ.forEach(t),Ev=i(Id,":"),Id.forEach(t),mm=f(e),d(fl.$$.fragment,e),cm=f(e),d(ml.$$.fragment,e),dm=f(e),rs=r(e,"P",{});var sJ=o(rs);Av=i(sJ,"We can see that this saves a few more GB on the GPU. Let\u2019s see how it looks when we add it to the other methods we introduced earlier:"),sJ.forEach(t),um=f(e),d(cl.$$.fragment,e),wm=f(e),d(dl.$$.fragment,e),ym=f(e),os=r(e,"P",{});var iJ=o(os);jv=i(iJ,"We went from 15 GB memory usage to 5 GB - a 3x improvement while maintaining the throughput! However, as mentioned before, the convergence of Adafactor can be worse than Adam. There is an alternative to Adafactor called 8-bit Adam that takes a slightly different approach."),iJ.forEach(t),bm=f(e),Je=r(e,"H3",{class:!0});var kd=o(Je);wt=r(kd,"A",{id:!0,class:!0,href:!0});var nJ=o(wt);Fn=r(nJ,"SPAN",{});var hJ=o(Fn);d(ul.$$.fragment,hJ),hJ.forEach(t),nJ.forEach(t),Iv=f(kd),xn=r(kd,"SPAN",{});var pJ=o(xn);kv=i(pJ,"8-bit Adam"),pJ.forEach(t),kd.forEach(t),Mm=f(e),ss=r(e,"P",{});var fJ=o(ss);$v=i(fJ,"Instead of aggregating optimizer states like Adafactor, 8-bit Adam keeps the full state and quantizes it. Quantization means that it stores the state with lower precision and dequantizes it only for the optimization. This is similar to the idea behind FP16 training where using variables with lower precision saves memory."),fJ.forEach(t),vm=f(e),j=r(e,"P",{});var ae=o(j);zv=i(ae,"In contrast to the previous approaches is this one not integrated into the "),is=r(ae,"A",{href:!0});var mJ=o(is);Cv=i(mJ,"Trainer"),mJ.forEach(t),Gv=i(ae," as a simple flag. We need to install the 8-bit optimizer and then pass it as a custom optimizer to the "),ns=r(ae,"A",{href:!0});var cJ=o(ns);Sv=i(cJ,"Trainer"),cJ.forEach(t),Rv=i(ae,". Follow the installation guide in the Github "),wl=r(ae,"A",{href:!0,rel:!0});var dJ=o(wl);Pv=i(dJ,"repo"),dJ.forEach(t),Wv=i(ae," to install the "),Yn=r(ae,"CODE",{});var uJ=o(Yn);Bv=i(uJ,"bitsandbytes"),uJ.forEach(t),Nv=i(ae," library that implements the 8-bit Adam optimizer."),ae.forEach(t),gm=f(e),hs=r(e,"P",{});var wJ=o(hs);Qv=i(wJ,"Once installed, we just need to initialize the the optimizer. Although this looks like a considerable amount of work it actually just involves two steps: first we need to group the model\u2019s parameters into two groups where to one group we apply weight decay and to the other we don\u2019t. Usually, biases and layer norm parameters are not weight decayed. Then in a second step we just do some argument housekeeping to use the same parameters as the previously used AdamW optimizer."),wJ.forEach(t),Tm=f(e),d(yt.$$.fragment,e),Um=f(e),d(yl.$$.fragment,e),_m=f(e),bt=r(e,"P",{});var $d=o(bt);Zv=i($d,"We can now pass the custom optimizer as an argument to the "),Ln=r($d,"CODE",{});var yJ=o(Ln);Fv=i(yJ,"Trainer"),yJ.forEach(t),xv=i($d,":"),$d.forEach(t),Jm=f(e),d(bl.$$.fragment,e),Em=f(e),d(Ml.$$.fragment,e),Am=f(e),ps=r(e,"P",{});var bJ=o(ps);Yv=i(bJ,"We can see that we get a similar memory improvement as with Adafactor while keeping the full rolling average of the gradients. Let\u2019s repeat the experiment with the full settings:"),bJ.forEach(t),jm=f(e),d(vl.$$.fragment,e),Im=f(e),d(gl.$$.fragment,e),km=f(e),fs=r(e,"P",{});var MJ=o(fs);Lv=i(MJ,"Again, we get about a 3x memory improvement and even slightly higher throughput as using Adafactor. So we have seen how we can optimize the memory footprint of large models. The following plot summarizes all our experiments:"),MJ.forEach(t),$m=f(e),ms=r(e,"P",{});var vJ=o(ms);cs=r(vJ,"IMG",{src:!0,alt:!0}),vJ.forEach(t),zm=f(e),Ee=r(e,"H3",{class:!0});var zd=o(Ee);Mt=r(zd,"A",{id:!0,class:!0,href:!0});var gJ=o(Mt);Dn=r(gJ,"SPAN",{});var TJ=o(Dn);d(Tl.$$.fragment,TJ),TJ.forEach(t),gJ.forEach(t),Dv=f(zd),Xn=r(zd,"SPAN",{});var UJ=o(Xn);Vn=r(UJ,"CODE",{});var _J=o(Vn);Xv=i(_J,"_multi_tensor"),_J.forEach(t),UJ.forEach(t),zd.forEach(t),Cm=i(e,`

pytorch-nightly introduced \`torch.optim._multi_tensor\` which should significantly speed up the optimizers for situations with lots of small feature tensors. It should eventually become the default, but if you want to experiment with it sooner and don't mind using the bleed-edge, see: https://github.com/huggingface/transformers/issues/9965
`),Ae=r(e,"H2",{class:!0});var Cd=o(Ae);vt=r(Cd,"A",{id:!0,class:!0,href:!0});var JJ=o(vt);Hn=r(JJ,"SPAN",{});var EJ=o(Hn);d(Ul.$$.fragment,EJ),EJ.forEach(t),JJ.forEach(t),Vv=f(Cd),On=r(Cd,"SPAN",{});var AJ=o(On);Hv=i(AJ,"Using \u{1F917} Accelerate"),AJ.forEach(t),Cd.forEach(t),Gm=f(e),q=r(e,"P",{});var ui=o(q);Ov=i(ui,"So far we have used the "),ds=r(ui,"A",{href:!0});var jJ=o(ds);qv=i(jJ,"Trainer"),jJ.forEach(t),Kv=i(ui," to run the experiments but a more flexible alternative to that approach is to use \u{1F917} Accelerate. With \u{1F917} Accelerate you have full control over the training loop and can essentially write the loop in pure PyTorch with some minor modifications. In turn it allows you to easily scale across different infrastructures such as CPUs, GPUs, TPUs, or distributed multi-GPU setups without changing any code. Let\u2019s see what it takes to implement all of the above tweaks in \u{1F917} Accelerate. We can still use the "),us=r(ui,"A",{href:!0});var IJ=o(us);eg=i(IJ,"TrainingArguments"),IJ.forEach(t),tg=i(ui," to wrap the training settings:"),ui.forEach(t),Sm=f(e),d(_l.$$.fragment,e),Rm=f(e),ws=r(e,"P",{});var kJ=o(ws);ag=i(kJ,"The full example training loop with \u{1F917} Accelerate is only a handful of lines of code long:"),kJ.forEach(t),Pm=f(e),d(Jl.$$.fragment,e),Wm=f(e),J=r(e,"P",{});var W=o(J);lg=i(W,"First we wrap the dataset in a "),El=r(W,"A",{href:!0,rel:!0});var $J=o(El);qn=r($J,"CODE",{});var zJ=o(qn);rg=i(zJ,"DataLoader"),zJ.forEach(t),$J.forEach(t),og=i(W,". Then we can enable gradient checkpointing by calling the model\u2019s "),ys=r(W,"A",{href:!0});var CJ=o(ys);sg=i(CJ,"gradient_checkpointing_enable()"),CJ.forEach(t),ig=i(W," method. When we initialize the "),Al=r(W,"A",{href:!0,rel:!0});var GJ=o(Al);Kn=r(GJ,"CODE",{});var SJ=o(Kn);ng=i(SJ,"Accelerator"),SJ.forEach(t),GJ.forEach(t),hg=i(W," we can specify if we want to use mixed precision training and it will take care of it for us in the "),eh=r(W,"CODE",{});var RJ=o(eh);pg=i(RJ,"prepare"),RJ.forEach(t),fg=i(W," call. During the "),jl=r(W,"A",{href:!0,rel:!0});var PJ=o(jl);th=r(PJ,"CODE",{});var WJ=o(th);mg=i(WJ,"prepare"),WJ.forEach(t),PJ.forEach(t),cg=i(W," call the dataloader will also be distributed across workers should we use multiple GPUs. We use the same 8-bit optimizer from the earlier experiments."),W.forEach(t),Bm=f(e),gt=r(e,"P",{});var Gd=o(gt);dg=i(Gd,"Finally, we can write the main training loop. Note that the "),ah=r(Gd,"CODE",{});var BJ=o(ah);ug=i(BJ,"backward"),BJ.forEach(t),wg=i(Gd," call is handled by \u{1F917} Accelerate. We can also see how gradient accumulation works: we normalize the loss so we get the average at the end of accumulation and once we have enough steps we run the optimization. Now the question is: does this use the same amount of memory as the previous steps? Let\u2019s check:"),Gd.forEach(t),Nm=f(e),d(Il.$$.fragment,e),Qm=f(e),Tt=r(e,"P",{});var Sd=o(Tt);yg=i(Sd,"Indeed it does. Implementing these optimization techniques with \u{1F917} Accelerate only takes a handful of lines of code and comes with the benefit of more flexiblity in the training loop. For a full documentation of all features have a look at the "),kl=r(Sd,"A",{href:!0,rel:!0});var NJ=o(kl);bg=i(NJ,"Accelerate documentation"),NJ.forEach(t),Mg=i(Sd,"."),Sd.forEach(t),Zm=f(e),je=r(e,"H2",{class:!0});var Rd=o(je);Ut=r(Rd,"A",{id:!0,class:!0,href:!0});var QJ=o(Ut);lh=r(QJ,"SPAN",{});var ZJ=o(lh);d($l.$$.fragment,ZJ),ZJ.forEach(t),QJ.forEach(t),vg=f(Rd),rh=r(Rd,"SPAN",{});var FJ=o(rh);gg=i(FJ,"DataLoader"),FJ.forEach(t),Rd.forEach(t),Fm=f(e),bs=r(e,"P",{});var xJ=o(bs);Tg=i(xJ,"One of the important requirements to reach great training speed is the ability to feed the GPU at the maximum speed it can handle. By default everything happens in the main process and it might not be able to read the data from disk fast enough, and thus create a bottleneck, leading to GPU under-utilization."),xJ.forEach(t),xm=f(e),_t=r(e,"UL",{});var Pd=o(_t);Ms=r(Pd,"LI",{});var j2=o(Ms);oh=r(j2,"CODE",{});var YJ=o(oh);Ug=i(YJ,"DataLoader(pin_memory=True, ...)"),YJ.forEach(t),_g=i(j2," which ensures that the data gets preloaded into the pinned memory on CPU and typically leads to much faster transfers from CPU to GPU memory."),j2.forEach(t),Jg=f(Pd),vs=r(Pd,"LI",{});var I2=o(vs);sh=r(I2,"CODE",{});var LJ=o(sh);Eg=i(LJ,"DataLoader(num_workers=4, ...)"),LJ.forEach(t),Ag=i(I2," - spawn several workers to pre-load data faster - during training watch the GPU utilization stats and if it\u2019s far from 100% experiment with raising the number of workers. Of course, the problem could be elsewhere so a very big number of workers won\u2019t necessarily lead to a better performance."),I2.forEach(t),Pd.forEach(t),Ym=f(e),Ie=r(e,"H2",{class:!0});var Wd=o(Ie);Jt=r(Wd,"A",{id:!0,class:!0,href:!0});var DJ=o(Jt);ih=r(DJ,"SPAN",{});var XJ=o(ih);d(zl.$$.fragment,XJ),XJ.forEach(t),DJ.forEach(t),jg=f(Wd),nh=r(Wd,"SPAN",{});var VJ=o(nh);Ig=i(VJ,"DeepSpeed ZeRO"),VJ.forEach(t),Wd.forEach(t),Lm=f(e),Et=r(e,"P",{});var Bd=o(Et);kg=i(Bd,"The in-depth details on how to use Deepspeed can be found "),gs=r(Bd,"A",{href:!0});var HJ=o(gs);$g=i(HJ,"here"),HJ.forEach(t),zg=i(Bd,"."),Bd.forEach(t),Dm=f(e),Ts=r(e,"P",{});var OJ=o(Ts);Cg=i(OJ,"First, a quick decision tree:"),OJ.forEach(t),Xm=f(e),At=r(e,"OL",{});var Nd=o(At);hh=r(Nd,"LI",{});var qJ=o(hh);Gg=i(qJ,"Model fits onto a single GPU and you have enough space to fit a small batch size - you don\u2019t need to use Deepspeed as it\u2019ll only slow things down in this use case."),qJ.forEach(t),Sg=f(Nd),ph=r(Nd,"LI",{});var KJ=o(ph);Rg=i(KJ,"Model doesn\u2019t fit onto a single GPU or you can\u2019t fit a small batch - use DeepSpeed ZeRO + CPU Offload and for much larger models NVMe Offload."),KJ.forEach(t),Nd.forEach(t),Vm=f(e),jt=r(e,"P",{});var Qd=o(jt);Pg=i(Qd,"Now if the decision tree suggested you use DeepSpeed first you need to "),Us=r(Qd,"A",{href:!0});var eE=o(Us);Wg=i(eE,"install it"),eE.forEach(t),Bg=i(Qd,", then follow one of the following guides to create a configuration file and launch DeepSpeed."),Qd.forEach(t),Hm=f(e),_s=r(e,"P",{});var tE=o(_s);Ng=i(tE,"Activation:"),tE.forEach(t),Om=f(e),G=r(e,"UL",{});var Dt=o(G);fh=r(Dt,"LI",{});var aE=o(fh);Cl=r(aE,"P",{});var Zd=o(Cl);Qg=i(Zd,"HF Trainer-based examples: see this "),Js=r(Zd,"A",{href:!0});var lE=o(Js);Zg=i(lE,"guide"),lE.forEach(t),Fg=i(Zd,"."),Zd.forEach(t),aE.forEach(t),xg=f(Dt),Gl=r(Dt,"LI",{});var Fd=o(Gl);mh=r(Fd,"P",{});var rE=o(mh);Yg=i(rE,"Custom HF Trainer-based program: Same as above, but pass:"),rE.forEach(t),Lg=f(Fd),d(Sl.$$.fragment,Fd),Fd.forEach(t),Dg=f(Dt),ch=r(Dt,"LI",{});var oE=o(ch);Rl=r(oE,"P",{});var xd=o(Rl);Xg=i(xd,"Deployment in Notebooks: see this "),Es=r(xd,"A",{href:!0});var sE=o(Es);Vg=i(sE,"guide"),sE.forEach(t),Hg=i(xd,"."),xd.forEach(t),oE.forEach(t),Og=f(Dt),dh=r(Dt,"LI",{});var iE=o(dh);ke=r(iE,"P",{});var wi=o(ke);qg=i(wi,"Custom training loop: This is somewhat complex but you can study how this is implemented in "),Pl=r(wi,"A",{href:!0,rel:!0});var nE=o(Pl);Kg=i(nE,"HF Trainer"),nE.forEach(t),e1=i(wi," - simply search for "),uh=r(wi,"CODE",{});var hE=o(uh);t1=i(hE,"deepspeed"),hE.forEach(t),a1=i(wi," in the code."),wi.forEach(t),iE.forEach(t),Dt.forEach(t),qm=f(e),$e=r(e,"H2",{class:!0});var Yd=o($e);It=r(Yd,"A",{id:!0,class:!0,href:!0});var pE=o(It);wh=r(pE,"SPAN",{});var fE=o(wh);d(Wl.$$.fragment,fE),fE.forEach(t),pE.forEach(t),l1=f(Yd),yh=r(Yd,"SPAN",{});var mE=o(yh);r1=i(mE,"Choice of GPU"),mE.forEach(t),Yd.forEach(t),Km=i(e,`

Sometimes, even when applying all the above tweaks the throughput on a given GPU might still not be good enough. One easy solution is to change the type of GPU. For example switching from let's say a K80 (which you typically get on Google Colab) to a fancier GPU such as the V100 or A100. Although they are more expensive they are usually more cost effective than cheaper GPUs due to their larger memory and faster architecture.
`),As=r(e,"P",{});var cE=o(As);o1=i(cE,"Now, let\u2019s take a step back and discuss what we should optimize for when scaling the training of large models."),cE.forEach(t),ec=f(e),ze=r(e,"H2",{class:!0});var Ld=o(ze);kt=r(Ld,"A",{id:!0,class:!0,href:!0});var dE=o(kt);bh=r(dE,"SPAN",{});var uE=o(bh);d(Bl.$$.fragment,uE),uE.forEach(t),dE.forEach(t),s1=f(Ld),Mh=r(Ld,"SPAN",{});var wE=o(Mh);i1=i(wE,"How to scale"),wE.forEach(t),Ld.forEach(t),tc=f(e),js=r(e,"P",{});var yE=o(js);n1=i(yE,"When we train models there are a two aspects we want to optimize at the same time:"),yE.forEach(t),ac=f(e),$t=r(e,"UL",{});var Dd=o($t);vh=r(Dd,"LI",{});var bE=o(vh);h1=i(bE,"Data throughput/training time"),bE.forEach(t),p1=f(Dd),gh=r(Dd,"LI",{});var ME=o(gh);f1=i(ME,"Model performance"),ME.forEach(t),Dd.forEach(t),lc=f(e),Is=r(e,"P",{});var vE=o(Is);m1=i(vE,"We have seen that each method changes the memory usage and throughput. In general we want to maximize the throughput (samples/second) to minimize the training cost. This is generally achieved by utilizing the GPU as much as possible and thus filling GPU memory to its limit. For example, as mentioned earlier, we only employ gradient accumulation when we want to use a batch size beyond the size of the GPU memory. If the desired batch size fits into memory then there is no reason to apply gradient accumulation which will only slow down training."),vE.forEach(t),rc=f(e),ks=r(e,"P",{});var gE=o(ks);c1=i(gE,"The second objective is model performance. Just because we can does not mean we should use a large batch size. As part of hyperparameter tuning you should determine which batch size yields the best result and then optimize the throughput accordingly."),gE.forEach(t),oc=f(e),Ce=r(e,"H2",{class:!0});var Xd=o(Ce);zt=r(Xd,"A",{id:!0,class:!0,href:!0});var TE=o(zt);Th=r(TE,"SPAN",{});var UE=o(Th);d(Nl.$$.fragment,UE),UE.forEach(t),TE.forEach(t),d1=f(Xd),Uh=r(Xd,"SPAN",{});var _E=o(Uh);u1=i(_E,"Efficient Software Prebuilds"),_E.forEach(t),Xd.forEach(t),sc=f(e),Ct=r(e,"P",{});var Vd=o(Ct);w1=i(Vd,"PyTorch\u2019s "),Ql=r(Vd,"A",{href:!0,rel:!0});var JE=o(Ql);y1=i(JE,"pip and conda builds"),JE.forEach(t),b1=i(Vd," come prebuit with the cuda toolkit which is enough to run PyTorch, but it is insufficient if you need to build cuda extensions."),Vd.forEach(t),ic=f(e),Gt=r(e,"P",{});var Hd=o(Gt);M1=i(Hd,"At times it may take an additional effort to pre-build some components, e.g., if you\u2019re using libraries like "),_h=r(Hd,"CODE",{});var EE=o(_h);v1=i(EE,"apex"),EE.forEach(t),g1=i(Hd," that don\u2019t come pre-compiled. In other situations figuring out how to install the right cuda toolkit system-wide can be complicated. To address these users\u2019 needs PyTorch and NVIDIA release a new version of NGC docker container which already comes with everything prebuilt and you just need to install your programs on it and it will run out of the box."),Hd.forEach(t),nc=f(e),$s=r(e,"P",{});var AE=o($s);T1=i(AE,"This approach is also useful if you want to tweak the pytorch source and/or make a new customized build."),AE.forEach(t),hc=f(e),K=r(e,"P",{});var yi=o(K);U1=i(yi,"To find the docker image version you want start "),Zl=r(yi,"A",{href:!0,rel:!0});var jE=o(Zl);_1=i(jE,"here"),jE.forEach(t),J1=i(yi,", choose one of the latest monthly releases. Go into the release\u2019s notes for the desired release, check that the environment\u2019s components are matching your needs (including NVIDIA Driver requirements!) and then at the very top of that document go to the corresponding NGC page. If for some reason you get lost, here is "),Fl=r(yi,"A",{href:!0,rel:!0});var IE=o(Fl);E1=i(IE,"the index of all PyTorch NGC images"),IE.forEach(t),A1=i(yi,"."),yi.forEach(t),pc=f(e),zs=r(e,"P",{});var kE=o(zs);j1=i(kE,"Next follow the instructions to download and deploy the docker image."),kE.forEach(t),fc=f(e),Ge=r(e,"H2",{class:!0});var Od=o(Ge);St=r(Od,"A",{id:!0,class:!0,href:!0});var $E=o(St);Jh=r($E,"SPAN",{});var zE=o(Jh);d(xl.$$.fragment,zE),zE.forEach(t),$E.forEach(t),I1=f(Od),Eh=r(Od,"SPAN",{});var CE=o(Eh);k1=i(CE,"Sparsity"),CE.forEach(t),Od.forEach(t),mc=f(e),Se=r(e,"H3",{class:!0});var qd=o(Se);Rt=r(qd,"A",{id:!0,class:!0,href:!0});var GE=o(Rt);Ah=r(GE,"SPAN",{});var SE=o(Ah);d(Yl.$$.fragment,SE),SE.forEach(t),GE.forEach(t),$1=f(qd),jh=r(qd,"SPAN",{});var RE=o(jh);z1=i(RE,"Mixture of Experts"),RE.forEach(t),qd.forEach(t),cc=f(e),Cs=r(e,"P",{});var PE=o(Cs);C1=i(PE,`Quite a few of the recent papers reported a 4-5x training speedup and a faster inference by integrating
Mixture of Experts (MoE) into the Transformer models.`),PE.forEach(t),dc=f(e),Gs=r(e,"P",{});var WE=o(Gs);G1=i(WE,"Since it has been discovered that more parameters lead to better performance, this technique allows to increase the number of parameters by an order of magnitude without increasing training costs."),WE.forEach(t),uc=f(e),Ss=r(e,"P",{});var BE=o(Ss);S1=i(BE,"In this approach every other FFN layer is replaced with a MoE Layer which consists of many experts, with a gated function that trains each expert in a balanced way depending on the input token\u2019s position in a sequence."),BE.forEach(t),wc=f(e),Rs=r(e,"P",{});var NE=o(Rs);Ps=r(NE,"IMG",{src:!0,alt:!0}),NE.forEach(t),yc=f(e),Pt=r(e,"P",{});var Kd=o(Pt);R1=i(Kd,"(source: "),Ll=r(Kd,"A",{href:!0,rel:!0});var QE=o(Ll);P1=i(QE,"GLAM"),QE.forEach(t),W1=i(Kd,")"),Kd.forEach(t),bc=f(e),Ws=r(e,"P",{});var ZE=o(Ws);B1=i(ZE,"You can find exhaustive details and comparison tables in the papers listed at the end of this section."),ZE.forEach(t),Mc=f(e),Bs=r(e,"P",{});var FE=o(Bs);N1=i(FE,"The main drawback of this approach is that it requires staggering amounts of GPU memory - almost an order of magnitude larger than its dense equivalent. Various distillation and approaches are proposed to how to overcome the much higher memory requirements."),FE.forEach(t),vc=f(e),Ns=r(e,"P",{});var xE=o(Ns);Q1=i(xE,"There is direct trade-off though, you can use just a few experts with a 2-3x smaller base model instead of dozens or hundreds experts leading to a 5x smaller model and thus increase the training speed moderately while increasing the memory requirements moderately as well."),xE.forEach(t),gc=f(e),Qs=r(e,"P",{});var YE=o(Qs);Z1=i(YE,"Most related papers and implementations are built around Tensorflow/TPUs:"),YE.forEach(t),Tc=f(e),ee=r(e,"UL",{});var bi=o(ee);Ih=r(bi,"LI",{});var LE=o(Ih);Dl=r(LE,"A",{href:!0,rel:!0});var DE=o(Dl);F1=i(DE,"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding"),DE.forEach(t),LE.forEach(t),x1=f(bi),kh=r(bi,"LI",{});var XE=o(kh);Xl=r(XE,"A",{href:!0,rel:!0});var VE=o(Xl);Y1=i(VE,"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity"),VE.forEach(t),XE.forEach(t),L1=f(bi),$h=r(bi,"LI",{});var HE=o($h);Vl=r(HE,"A",{href:!0,rel:!0});var OE=o(Vl);D1=i(OE,"GLaM: Generalist Language Model (GLaM)"),OE.forEach(t),HE.forEach(t),bi.forEach(t),Uc=f(e),T=r(e,"P",{});var k=o(T);X1=i(k,"And for Pytorch DeepSpeed has built one as well: "),Hl=r(k,"A",{href:!0,rel:!0});var qE=o(Hl);V1=i(qE,"DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale"),qE.forEach(t),H1=i(k,", "),Ol=r(k,"A",{href:!0,rel:!0});var KE=o(Ol);O1=i(KE,"Mixture of Experts"),KE.forEach(t),q1=i(k," - blog posts:  "),ql=r(k,"A",{href:!0,rel:!0});var e3=o(ql);K1=i(e3,"1"),e3.forEach(t),eT=i(k,", "),Kl=r(k,"A",{href:!0,rel:!0});var t3=o(Kl);tT=i(t3,"2"),t3.forEach(t),aT=i(k," and specific deployment with large transformer-based natural language generation models: "),er=r(k,"A",{href:!0,rel:!0});var a3=o(er);lT=i(a3,"blog post"),a3.forEach(t),rT=i(k,", "),Zs=r(k,"A",{href:!0});var l3=o(Zs);oT=i(l3,"Megatron-Deepspeed branch"),l3.forEach(t),sT=i(k,"."),k.forEach(t),_c=f(e),Re=r(e,"H2",{class:!0});var eu=o(Re);Wt=r(eu,"A",{id:!0,class:!0,href:!0});var r3=o(Wt);zh=r(r3,"SPAN",{});var o3=o(zh);d(tr.$$.fragment,o3),o3.forEach(t),r3.forEach(t),iT=f(eu),Ch=r(eu,"SPAN",{});var s3=o(Ch);nT=i(s3,"Scaling beyond a single GPU"),s3.forEach(t),eu.forEach(t),Jc=f(e),Fs=r(e,"P",{});var i3=o(Fs);hT=i(i3,"For some applications, such as pretraining large language models, applying all the approaches above might still not be fast enough. In this case you want to scale your experiment to several GPUs."),i3.forEach(t),Ec=f(e),Bt=r(e,"P",{});var tu=o(Bt);pT=i(tu,"Another use case for training on many GPUs is if the model does not fit on a single GPU with all the mentioned tricks. There are still more methods we can apply although life starts to get a bit more complicated. This usually involves some form of pipeline or tensor parallelism where the model itself is distributed across several GPUs. One can also make use of DeepSpeed which implements some of these parallelism strategies along with some more optimization to reduce the memory footprint such as partitioning the optimizer states. You can read more about this in the "),xs=r(tu,"A",{href:!0});var n3=o(xs);fT=i(n3,"\u201CMulti-GPU training\u201D section"),n3.forEach(t),mT=i(tu,"."),tu.forEach(t),Ac=f(e),Pe=r(e,"H2",{class:!0});var au=o(Pe);Nt=r(au,"A",{id:!0,class:!0,href:!0});var h3=o(Nt);Gh=r(h3,"SPAN",{});var p3=o(Gh);d(ar.$$.fragment,p3),p3.forEach(t),h3.forEach(t),cT=f(au),Sh=r(au,"SPAN",{});var f3=o(Sh);dT=i(f3,"Using PyTorch native attention"),f3.forEach(t),au.forEach(t),jc=f(e),S=r(e,"P",{});var Xt=o(S);uT=i(Xt,"PyTorch 2.0 released the native "),lr=r(Xt,"A",{href:!0,rel:!0});var m3=o(lr);Rh=r(m3,"CODE",{});var c3=o(Rh);wT=i(c3,"torch.nn.functional.scaled_dot_product_attention"),c3.forEach(t),m3.forEach(t),yT=i(Xt," (SDPA), that allows to use fused GPU kernels as "),rr=r(Xt,"A",{href:!0,rel:!0});var d3=o(rr);bT=i(d3,"memory-efficient attention"),d3.forEach(t),MT=i(Xt," and "),or=r(Xt,"A",{href:!0,rel:!0});var u3=o(or);vT=i(u3,"flash attention"),u3.forEach(t),gT=i(Xt,"."),Xt.forEach(t),Ic=f(e),Qt=r(e,"P",{});var lu=o(Qt);TT=i(lu,"After installing the "),sr=r(lu,"A",{href:!0,rel:!0});var w3=o(sr);Ph=r(w3,"CODE",{});var y3=o(Ph);UT=i(y3,"optimum"),y3.forEach(t),w3.forEach(t),_T=i(lu," package, the relevant internal modules can be replaced to use PyTorch\u2019s native attention with:"),lu.forEach(t),kc=f(e),d(ir.$$.fragment,e),$c=f(e),Ys=r(e,"P",{});var b3=o(Ys);JT=i(b3,"Training can then be done as usual."),b3.forEach(t),zc=f(e),We=r(e,"H2",{class:!0});var ru=o(We);Zt=r(ru,"A",{id:!0,class:!0,href:!0});var M3=o(Zt);Wh=r(M3,"SPAN",{});var v3=o(Wh);d(nr.$$.fragment,v3),v3.forEach(t),M3.forEach(t),ET=f(ru),Bh=r(ru,"SPAN",{});var g3=o(Bh);AT=i(g3,"Using torch.compile"),g3.forEach(t),ru.forEach(t),Cc=f(e),Ft=r(e,"P",{});var ou=o(Ft);jT=i(ou,"PyTorch 2.0 introduces a new compile function, you can learn more about it "),hr=r(ou,"A",{href:!0,rel:!0});var T3=o(hr);IT=i(T3,"in their documentation"),T3.forEach(t),kT=i(ou,". It uses Python\u2019s frame evaluation API to automatically create a graph from existing PyTorch programs. After capturing the graph, different backends can be deployed to lower the graph to an optimized engine. You can choose one option below for performance boost."),ou.forEach(t),Gc=f(e),B=r(e,"P",{});var $r=o(B);Nh=r($r,"CODE",{});var U3=o(Nh);$T=i(U3,"torch.compile"),U3.forEach(t),zT=i($r," has a growing list of backends, which can be found in "),pr=r($r,"A",{href:!0,rel:!0});var _3=o(pr);CT=i(_3,"backends.py"),_3.forEach(t),GT=i($r,`
or `),Qh=r($r,"CODE",{});var J3=o(Qh);ST=i(J3,"torchdynamo.list_backends()"),J3.forEach(t),RT=i($r," each of which with its optional dependencies."),$r.forEach(t),Sc=f(e),Ls=r(e,"P",{});var E3=o(Ls);PT=i(E3,"Some of the most commonly used backends are"),E3.forEach(t),Rc=f(e),fr=r(e,"P",{});var k2=o(fr);Zh=r(k2,"STRONG",{});var A3=o(Zh);WT=i(A3,"Debugging backends"),A3.forEach(t),BT=i(k2,":"),k2.forEach(t),Pc=f(e),xt=r(e,"UL",{});var su=o(xt);Ds=r(su,"LI",{});var $2=o(Ds);Fh=r($2,"CODE",{});var j3=o(Fh);NT=i(j3,'dynamo.optimize("eager")'),j3.forEach(t),QT=i($2," - Uses PyTorch to run the extracted GraphModule. This is quite useful in debugging TorchDynamo issues."),$2.forEach(t),ZT=f(su),Xs=r(su,"LI",{});var z2=o(Xs);xh=r(z2,"CODE",{});var I3=o(xh);FT=i(I3,'dynamo.optimize("aot_eager")'),I3.forEach(t),xT=i(z2," - Uses AotAutograd with no compiler, i.e, just using PyTorch eager for the AotAutograd\u2019s extracted forward and backward graphs. This is useful for debugging, and unlikely to give speedups."),z2.forEach(t),su.forEach(t),Wc=f(e),mr=r(e,"P",{});var C2=o(mr);Yh=r(C2,"STRONG",{});var k3=o(Yh);YT=i(k3,"Training & inference backends"),k3.forEach(t),LT=i(C2,":"),C2.forEach(t),Bc=f(e),R=r(e,"UL",{});var Vt=o(R);cr=r(Vt,"LI",{});var iu=o(cr);Lh=r(iu,"CODE",{});var $3=o(Lh);DT=i($3,'dynamo.optimize("inductor")'),$3.forEach(t),XT=i(iu," - Uses TorchInductor backend with AotAutograd and cudagraphs by leveraging codegened Triton kernels  "),dr=r(iu,"A",{href:!0,rel:!0});var z3=o(dr);VT=i(z3,"Read more"),z3.forEach(t),iu.forEach(t),HT=f(Vt),ur=r(Vt,"LI",{});var nu=o(ur);Dh=r(nu,"CODE",{});var C3=o(Dh);OT=i(C3,'dynamo.optimize("nvfuser")'),C3.forEach(t),qT=i(nu," -  nvFuser with TorchScript. "),wr=r(nu,"A",{href:!0,rel:!0});var G3=o(wr);KT=i(G3,"Read more"),G3.forEach(t),nu.forEach(t),e2=f(Vt),yr=r(Vt,"LI",{});var hu=o(yr);Xh=r(hu,"CODE",{});var S3=o(Xh);t2=i(S3,'dynamo.optimize("aot_nvfuser")'),S3.forEach(t),a2=i(hu," -  nvFuser with AotAutograd. "),br=r(hu,"A",{href:!0,rel:!0});var R3=o(br);l2=i(R3,"Read more"),R3.forEach(t),hu.forEach(t),r2=f(Vt),Mr=r(Vt,"LI",{});var pu=o(Mr);Vh=r(pu,"CODE",{});var P3=o(Vh);o2=i(P3,'dynamo.optimize("aot_cudagraphs")'),P3.forEach(t),s2=i(pu," - cudagraphs with AotAutograd. "),vr=r(pu,"A",{href:!0,rel:!0});var W3=o(vr);i2=i(W3,"Read more"),W3.forEach(t),pu.forEach(t),Vt.forEach(t),Nc=f(e),gr=r(e,"P",{});var G2=o(gr);Hh=r(G2,"STRONG",{});var B3=o(Hh);n2=i(B3,"Inference-only backend"),B3.forEach(t),h2=i(G2,"s:"),G2.forEach(t),Qc=f(e),P=r(e,"UL",{});var Ht=o(P);Tr=r(Ht,"LI",{});var fu=o(Tr);Oh=r(fu,"CODE",{});var N3=o(Oh);p2=i(N3,'dynamo.optimize("ofi")'),N3.forEach(t),f2=i(fu," -  Uses Torchscript optimize_for_inference.  "),Ur=r(fu,"A",{href:!0,rel:!0});var Q3=o(Ur);m2=i(Q3,"Read more"),Q3.forEach(t),fu.forEach(t),c2=f(Ht),_r=r(Ht,"LI",{});var mu=o(_r);qh=r(mu,"CODE",{});var Z3=o(qh);d2=i(Z3,'dynamo.optimize("fx2trt")'),Z3.forEach(t),u2=i(mu," -  Uses Nvidia TensorRT for inference optimizations.  "),Jr=r(mu,"A",{href:!0,rel:!0});var F3=o(Jr);w2=i(F3,"Read more"),F3.forEach(t),mu.forEach(t),y2=f(Ht),Er=r(Ht,"LI",{});var cu=o(Er);Kh=r(cu,"CODE",{});var x3=o(Kh);b2=i(x3,'dynamo.optimize("onnxrt")'),x3.forEach(t),M2=i(cu," -  Uses ONNXRT for inference on CPU/GPU.  "),Ar=r(cu,"A",{href:!0,rel:!0});var Y3=o(Ar);v2=i(Y3,"Read more"),Y3.forEach(t),cu.forEach(t),g2=f(Ht),jr=r(Ht,"LI",{});var du=o(jr);ep=r(du,"CODE",{});var L3=o(ep);T2=i(L3,'dynamo.optimize("ipex")'),L3.forEach(t),U2=i(du," -  Uses IPEX for inference on CPU.  "),Ir=r(du,"A",{href:!0,rel:!0});var D3=o(Ir);_2=i(D3,"Read more"),D3.forEach(t),du.forEach(t),Ht.forEach(t),this.h()},h(){m(g,"name","hf:doc:metadata"),m(g,"content",JSON.stringify(lA)),m($,"id","efficient-training-on-a-single-gpu"),m($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m($,"href","#efficient-training-on-a-single-gpu"),m(U,"class","relative group"),m(Cr,"href","perf_train_gpu_many"),m(Gr,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),m(qt,"href","https://huggingface.co/docs/accelerate/"),m(qt,"rel","nofollow"),m(Sr,"align","left"),m(Rr,"align","left"),m(Pr,"align","left"),m(Wr,"align","left"),m(Br,"align","left"),m(Nr,"align","left"),m(Qr,"align","left"),m(Zr,"align","left"),m(Fr,"align","left"),m(xr,"align","left"),m(Yr,"align","left"),m(Lr,"align","left"),m(Dr,"align","left"),m(Xr,"align","left"),m(Vr,"align","left"),m(Hr,"align","left"),m(Or,"align","left"),m(qr,"align","left"),m(Kr,"align","left"),m(eo,"align","left"),m(to,"align","left"),m(ao,"align","left"),m(lo,"align","left"),m(ro,"align","left"),m(ea,"href","https://huggingface.co/docs/datasets/v2.13.1/en/package_reference/main_classes#datasets.Dataset"),m(ea,"rel","nofollow"),m(so,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),m(Fe,"id","load-model"),m(Fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Fe,"href","#load-model"),m(fe,"class","relative group"),m(De,"id","vanilla-training"),m(De,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(De,"href","#vanilla-training"),m(me,"class","relative group"),m(fo,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),m(Ve,"id","anatomy-of-models-operations"),m(Ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ve,"href","#anatomy-of-models-operations"),m(ce,"class","relative group"),m(ga,"href","https://arxiv.org/abs/2007.00072"),m(ga,"rel","nofollow"),m(He,"id","anatomy-of-models-memory"),m(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(He,"href","#anatomy-of-models-memory"),m(de,"class","relative group"),m(Ua,"href","https://github.com/TimDettmers/bitsandbytes"),m(Ua,"rel","nofollow"),m(Ke,"id","batch-sizes"),m(Ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ke,"href","#batch-sizes"),m(ue,"class","relative group"),m(Ja,"href","https://docs.nvidia.com/deeplearning/performance/dl-performance-fully-connected/index.html#input-features"),m(Ja,"rel","nofollow"),m(Ea,"href","https://docs.nvidia.com/deeplearning/performance/dl-performance-fully-connected/index.html#batch-size"),m(Ea,"rel","nofollow"),m(ja,"href","https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc"),m(ja,"rel","nofollow"),m(Ia,"href","https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#dim-quantization"),m(Ia,"rel","nofollow"),m(tt,"id","gradient-accumulation"),m(tt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(tt,"href","#gradient-accumulation"),m(we,"class","relative group"),m(So,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),m(Ro,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.TrainingArguments"),m(Ca,"href","https://github.com/huggingface/transformers/issues/14608#issuecomment-1004392537"),m(Ca,"rel","nofollow"),m(Ga,"href","https://github.com/huggingface/transformers/issues/15026#issuecomment-1005033957"),m(Ga,"rel","nofollow"),m(at,"id","gradient-checkpointing"),m(at,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(at,"href","#gradient-checkpointing"),m(ye,"class","relative group"),m(Ra,"href","https://medium.com/tensorflow/fitting-larger-networks-into-memory-583e3c758ff9"),m(Ra,"rel","nofollow"),m(Bo,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),m(No,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.TrainingArguments"),m(rt,"id","floating-data-types"),m(rt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(rt,"href","#floating-data-types"),m(be,"class","relative group"),S2(xo.src,R2="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/tf32-bf16-fp16-fp32.png")||m(xo,"src",R2),m(xo,"alt","data types"),m(Fa,"href","https://developer.nvidia.com/blog/accelerating-ai-training-with-tf32-tensor-cores/"),m(Fa,"rel","nofollow"),m(ot,"id","fp16-training"),m(ot,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ot,"href","#fp16-training"),m(ve,"class","relative group"),m(st,"id","bf16"),m(st,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(st,"href","#bf16"),m(ge,"class","relative group"),m(it,"id","tf32"),m(it,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(it,"href","#tf32"),m(Te,"class","relative group"),m(Ka,"href","https://developer.nvidia.com/blog/accelerating-ai-training-with-tf32-tensor-cores/"),m(Ka,"rel","nofollow"),m(tl,"href","https://github.com/huggingface/transformers/issues/14608#issuecomment-1004390803"),m(tl,"rel","nofollow"),m(al,"href","https://github.com/huggingface/transformers/issues/15026#issuecomment-1004543189"),m(al,"rel","nofollow"),m(pt,"id","optimizer"),m(pt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(pt,"href","#optimizer"),m(Ue,"class","relative group"),m(ol,"href","https://github.com/NVIDIA/apex"),m(ol,"rel","nofollow"),m(sl,"href","https://github.com/TimDettmers/bitsandbytes"),m(sl,"rel","nofollow"),m(dt,"id","adafactor"),m(dt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(dt,"href","#adafactor"),m(_e,"class","relative group"),m(wt,"id","8bit-adam"),m(wt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(wt,"href","#8bit-adam"),m(Je,"class","relative group"),m(is,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),m(ns,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),m(wl,"href","https://github.com/TimDettmers/bitsandbytes"),m(wl,"rel","nofollow"),S2(cs.src,P2="https://huggingface.co/datasets/lvwerra/repo-images/raw/main/gpu-memory-savings.png")||m(cs,"src",P2),m(cs,"alt","png"),m(Mt,"id","multitensor"),m(Mt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Mt,"href","#multitensor"),m(Ee,"class","relative group"),m(vt,"id","using-accelerate"),m(vt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(vt,"href","#using-accelerate"),m(Ae,"class","relative group"),m(ds,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),m(us,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.TrainingArguments"),m(El,"href","https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader"),m(El,"rel","nofollow"),m(ys,"href","/docs/transformers/v4.31.0/en/main_classes/model#transformers.PreTrainedModel.gradient_checkpointing_enable"),m(Al,"href","https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator"),m(Al,"rel","nofollow"),m(jl,"href","https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.prepare"),m(jl,"rel","nofollow"),m(kl,"href","https://huggingface.co/docs/accelerate/index"),m(kl,"rel","nofollow"),m(Ut,"id","dataloader"),m(Ut,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ut,"href","#dataloader"),m(je,"class","relative group"),m(Jt,"id","deepspeed-zero"),m(Jt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Jt,"href","#deepspeed-zero"),m(Ie,"class","relative group"),m(gs,"href","main_classes/deepspeed"),m(Us,"href","main_classes/deepspeed#installation"),m(Js,"href","main_classes/deepspeed#deployment-with-one-gpu"),m(Es,"href","main_classes/deepspeed#deployment-in-notebooks"),m(Pl,"href","https://github.com/huggingface/transformers/blob/master/src/transformers/trainer.py"),m(Pl,"rel","nofollow"),m(It,"id","choice-of-gpu"),m(It,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(It,"href","#choice-of-gpu"),m($e,"class","relative group"),m(kt,"id","how-to-scale"),m(kt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(kt,"href","#how-to-scale"),m(ze,"class","relative group"),m(zt,"id","efficient-software-prebuilds"),m(zt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(zt,"href","#efficient-software-prebuilds"),m(Ce,"class","relative group"),m(Ql,"href","https://pytorch.org/get-started/locally/#start-locally"),m(Ql,"rel","nofollow"),m(Zl,"href","https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/"),m(Zl,"rel","nofollow"),m(Fl,"href","https://ngc.nvidia.com/catalog/containers/nvidia:pytorch"),m(Fl,"rel","nofollow"),m(St,"id","sparsity"),m(St,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(St,"href","#sparsity"),m(Ge,"class","relative group"),m(Rt,"id","mixture-of-experts"),m(Rt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Rt,"href","#mixture-of-experts"),m(Se,"class","relative group"),S2(Ps.src,W2="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/perf-moe-transformer.png")||m(Ps,"src",W2),m(Ps,"alt","MoE Transformer 2x block"),m(Ll,"href","https://ai.googleblog.com/2021/12/more-efficient-in-context-learning-with.html"),m(Ll,"rel","nofollow"),m(Dl,"href","https://arxiv.org/abs/2006.16668"),m(Dl,"rel","nofollow"),m(Xl,"href","https://arxiv.org/abs/2101.03961"),m(Xl,"rel","nofollow"),m(Vl,"href","https://ai.googleblog.com/2021/12/more-efficient-in-context-learning-with.html"),m(Vl,"rel","nofollow"),m(Hl,"href","https://arxiv.org/abs/2201.05596"),m(Hl,"rel","nofollow"),m(Ol,"href","https://www.deepspeed.ai/tutorials/mixture-of-experts/"),m(Ol,"rel","nofollow"),m(ql,"href","https://www.microsoft.com/en-us/research/blog/deepspeed-powers-8x-larger-moe-model-training-with-high-performance/"),m(ql,"rel","nofollow"),m(Kl,"href","https://www.microsoft.com/en-us/research/publication/scalable-and-efficient-moe-training-for-multitask-multilingual-models/"),m(Kl,"rel","nofollow"),m(er,"href","https://www.deepspeed.ai/news/2021/12/09/deepspeed-moe-nlg.html"),m(er,"rel","nofollow"),m(Zs,"href","Thttps://github.com/microsoft/Megatron-DeepSpeed/tree/moe-training"),m(Wt,"id","scaling-beyond-a-single-gpu"),m(Wt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Wt,"href","#scaling-beyond-a-single-gpu"),m(Re,"class","relative group"),m(xs,"href","perf_train_gpu_many"),m(Nt,"id","using-pytorch-native-attention"),m(Nt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Nt,"href","#using-pytorch-native-attention"),m(Pe,"class","relative group"),m(lr,"href","https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html"),m(lr,"rel","nofollow"),m(rr,"href","https://arxiv.org/abs/2112.05682"),m(rr,"rel","nofollow"),m(or,"href","https://arxiv.org/abs/2205.14135"),m(or,"rel","nofollow"),m(sr,"href","https://github.com/huggingface/optimum"),m(sr,"rel","nofollow"),m(Zt,"id","using-torchcompile"),m(Zt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Zt,"href","#using-torchcompile"),m(We,"class","relative group"),m(hr,"href","https://pytorch.org/get-started/pytorch-2.0/"),m(hr,"rel","nofollow"),m(pr,"href","https://github.com/pytorch/pytorch/blob/master/torch/_dynamo/optimizations/backends.py"),m(pr,"rel","nofollow"),m(dr,"href","https://dev-discuss.pytorch.org/t/torchinductor-a-pytorch-native-compiler-with-define-by-run-ir-and-symbolic-shapes/747"),m(dr,"rel","nofollow"),m(wr,"href","https://dev-discuss.pytorch.org/t/tracing-with-primitives-update-1-nvfuser-and-its-primitives/593"),m(wr,"rel","nofollow"),m(br,"href","https://dev-discuss.pytorch.org/t/tracing-with-primitives-update-1-nvfuser-and-its-primitives/593"),m(br,"rel","nofollow"),m(vr,"href","https://github.com/pytorch/torchdynamo/pull/757"),m(vr,"rel","nofollow"),m(Ur,"href","https://pytorch.org/docs/stable/generated/torch.jit.optimize_for_inference.html"),m(Ur,"rel","nofollow"),m(Jr,"href","https://github.com/pytorch/TensorRT/blob/master/docsrc/tutorials/getting_started_with_fx_path.rst"),m(Jr,"rel","nofollow"),m(Ar,"href","https://onnxruntime.ai/"),m(Ar,"rel","nofollow"),m(Ir,"href","https://github.com/intel/intel-extension-for-pytorch"),m(Ir,"rel","nofollow")},m(e,n){a(document.head,g),h(e,E,n),h(e,U,n),a(U,$),a($,Mi),u(Ot,Mi,null),a(U,uu),a(U,vi),a(vi,wu),h(e,rp,n),h(e,Be,n),a(Be,yu),a(Be,Cr),a(Cr,bu),a(Be,Mu),h(e,op,n),h(e,N,n),a(N,vu),a(N,Gr),a(Gr,gu),a(N,Tu),a(N,qt),a(qt,Uu),a(N,_u),h(e,sp,n),h(e,Ne,n),a(Ne,gi),a(gi,le),a(le,Sr),a(Sr,Ju),a(le,Eu),a(le,Rr),a(Rr,Au),a(le,ju),a(le,Pr),a(Pr,Iu),a(Ne,ku),a(Ne,_),a(_,re),a(re,Wr),a(Wr,$u),a(re,zu),a(re,Br),a(Br,Cu),a(re,Gu),a(re,Nr),a(Nr,Su),a(_,Ru),a(_,oe),a(oe,Qr),a(Qr,Pu),a(oe,Wu),a(oe,Zr),a(Zr,Bu),a(oe,Nu),a(oe,Fr),a(Fr,Qu),a(_,Zu),a(_,se),a(se,xr),a(xr,Fu),a(se,xu),a(se,Yr),a(Yr,Yu),a(se,Lu),a(se,Lr),a(Lr,Du),a(_,Xu),a(_,ie),a(ie,Dr),a(Dr,Vu),a(ie,Hu),a(ie,Xr),a(Xr,Ou),a(ie,qu),a(ie,Vr),a(Vr,Ku),a(_,ew),a(_,ne),a(ne,Hr),a(Hr,tw),a(ne,aw),a(ne,Or),a(Or,lw),a(ne,rw),a(ne,qr),a(qr,ow),a(_,sw),a(_,he),a(he,Kr),a(Kr,iw),a(he,nw),a(he,eo),a(eo,hw),a(he,pw),a(he,to),a(to,fw),a(_,mw),a(_,pe),a(pe,ao),a(ao,cw),a(pe,dw),a(pe,lo),a(lo,uw),a(pe,ww),a(pe,ro),a(ro,yw),h(e,ip,n),h(e,oo,n),a(oo,bw),h(e,np,n),u(Kt,e,n),h(e,hp,n),h(e,Q,n),a(Q,Mw),a(Q,Ti),a(Ti,vw),a(Q,gw),a(Q,Ui),a(Ui,Tw),a(Q,Uw),h(e,pp,n),h(e,Qe,n),a(Qe,_w),a(Qe,ea),a(ea,Jw),a(Qe,Ew),h(e,fp,n),u(ta,e,n),h(e,mp,n),h(e,Ze,n),a(Ze,Aw),a(Ze,so),a(so,jw),a(Ze,Iw),h(e,cp,n),u(aa,e,n),h(e,dp,n),h(e,io,n),a(io,kw),h(e,up,n),u(la,e,n),h(e,wp,n),h(e,no,n),a(no,$w),h(e,yp,n),u(ra,e,n),h(e,bp,n),h(e,ho,n),a(ho,zw),h(e,Mp,n),h(e,fe,n),a(fe,Fe),a(Fe,_i),u(oa,_i,null),a(fe,Cw),a(fe,Ji),a(Ji,Gw),h(e,vp,n),h(e,xe,n),a(xe,Sw),a(xe,Ei),a(Ei,Rw),a(xe,Pw),h(e,gp,n),u(sa,e,n),h(e,Tp,n),h(e,Ye,n),a(Ye,Ww),a(Ye,Ai),a(Ai,Bw),a(Ye,Nw),h(e,Up,n),u(ia,e,n),h(e,_p,n),u(na,e,n),h(e,Jp,n),h(e,po,n),a(po,Qw),h(e,Ep,n),u(ha,e,n),h(e,Ap,n),u(Le,e,n),h(e,jp,n),h(e,me,n),a(me,De),a(De,ji),u(pa,ji,null),a(me,Zw),a(me,Ii),a(Ii,Fw),h(e,Ip,n),h(e,Xe,n),a(Xe,xw),a(Xe,fo),a(fo,Yw),a(Xe,Lw),h(e,kp,n),u(fa,e,n),h(e,$p,n),u(ma,e,n),h(e,zp,n),h(e,mo,n),a(mo,Dw),h(e,Cp,n),h(e,ce,n),a(ce,Ve),a(Ve,ki),u(ca,ki,null),a(ce,Xw),a(ce,$i),a($i,Vw),h(e,Gp,n),h(e,co,n),a(co,Hw),h(e,Sp,n),h(e,Z,n),a(Z,da),a(da,zi),a(zi,Ci),a(Ci,Ow),a(da,qw),a(da,ua),a(ua,Kw),a(ua,Gi),a(Gi,ey),a(ua,ty),a(Z,ay),a(Z,wa),a(wa,Si),a(Si,Ri),a(Ri,ly),a(wa,ry),a(wa,ya),a(ya,oy),a(ya,Pi),a(Pi,sy),a(ya,iy),a(Z,ny),a(Z,ba),a(ba,Wi),a(Wi,Bi),a(Bi,hy),a(ba,py),a(ba,Ma),a(Ma,fy),a(Ma,Ni),a(Ni,my),a(Ma,cy),h(e,Rp,n),h(e,uo,n),a(uo,dy),h(e,Pp,n),h(e,va,n),a(va,uy),a(va,ga),a(ga,wy),h(e,Wp,n),h(e,de,n),a(de,He),a(He,Qi),u(Ta,Qi,null),a(de,yy),a(de,Zi),a(Zi,by),h(e,Bp,n),h(e,wo,n),a(wo,My),h(e,Np,n),h(e,yo,n),a(yo,vy),h(e,Qp,n),h(e,bo,n),a(bo,Fi),a(Fi,gy),h(e,Zp,n),h(e,Oe,n),a(Oe,xi),a(xi,Ty),a(Oe,Uy),a(Oe,Yi),a(Yi,_y),h(e,Fp,n),h(e,Mo,n),a(Mo,Li),a(Li,Jy),h(e,xp,n),h(e,F,n),a(F,Di),a(Di,Ey),a(F,Ay),a(F,vo),a(vo,jy),a(vo,Ua),a(Ua,Iy),a(F,ky),a(F,Xi),a(Xi,$y),h(e,Yp,n),h(e,go,n),a(go,Vi),a(Vi,zy),h(e,Lp,n),h(e,To,n),a(To,Hi),a(Hi,Cy),h(e,Dp,n),h(e,Uo,n),a(Uo,Oi),a(Oi,Gy),h(e,Xp,n),h(e,_o,n),a(_o,qi),a(qi,Sy),h(e,Vp,n),h(e,Jo,n),a(Jo,Ry),h(e,Hp,n),h(e,Eo,n),a(Eo,Ki),a(Ki,Py),h(e,Op,n),h(e,Ao,n),a(Ao,Wy),h(e,qp,n),h(e,jo,n),a(jo,en),a(en,By),h(e,Kp,n),h(e,Io,n),a(Io,Ny),h(e,ef,n),h(e,ko,n),a(ko,qe),a(qe,tn),a(tn,Qy),a(qe,Zy),a(qe,an),a(an,Fy),a(qe,xy),h(e,tf,n),h(e,$o,n),a($o,Yy),h(e,af,n),h(e,zo,n),a(zo,Ly),h(e,lf,n),h(e,ue,n),a(ue,Ke),a(Ke,ln),u(_a,ln,null),a(ue,Dy),a(ue,rn),a(rn,Xy),h(e,rf,n),h(e,Co,n),a(Co,Vy),h(e,of,n),h(e,x,n),a(x,Hy),a(x,Ja),a(Ja,Oy),a(x,qy),a(x,Ea),a(Ea,Ky),a(x,eb),h(e,sf,n),h(e,Aa,n),a(Aa,ja),a(ja,tb),a(Aa,ab),h(e,nf,n),h(e,et,n),a(et,lb),a(et,Ia),a(Ia,rb),a(et,ob),h(e,hf,n),h(e,we,n),a(we,tt),a(tt,on),u(ka,on,null),a(we,sb),a(we,sn),a(sn,ib),h(e,pf,n),h(e,Go,n),a(Go,nb),h(e,ff,n),h(e,z,n),a(z,hb),a(z,So),a(So,pb),a(z,fb),a(z,nn),a(nn,mb),a(z,cb),a(z,Ro),a(Ro,db),a(z,ub),h(e,mf,n),u($a,e,n),h(e,cf,n),u(za,e,n),h(e,df,n),h(e,A,n),a(A,wb),a(A,hn),a(hn,yb),a(A,bb),a(A,pn),a(pn,Mb),a(A,vb),a(A,fn),a(fn,gb),a(A,Tb),a(A,mn),a(mn,Ub),a(A,_b),h(e,uf,n),h(e,Y,n),a(Y,Jb),a(Y,Ca),a(Ca,Eb),a(Y,Ab),a(Y,Ga),a(Ga,jb),a(Y,Ib),h(e,wf,n),h(e,Po,n),a(Po,kb),h(e,yf,n),h(e,ye,n),a(ye,at),a(at,cn),u(Sa,cn,null),a(ye,$b),a(ye,dn),a(dn,zb),h(e,bf,n),h(e,Wo,n),a(Wo,Cb),h(e,Mf,n),h(e,lt,n),a(lt,Gb),a(lt,Ra),a(Ra,Sb),a(lt,Rb),h(e,vf,n),h(e,L,n),a(L,Pb),a(L,Bo),a(Bo,Wb),a(L,Bb),a(L,No),a(No,Nb),a(L,Qb),h(e,gf,n),u(Pa,e,n),h(e,Tf,n),u(Wa,e,n),h(e,Uf,n),h(e,Qo,n),a(Qo,Zb),h(e,_f,n),h(e,be,n),a(be,rt),a(rt,un),u(Ba,un,null),a(be,Fb),a(be,wn),a(wn,xb),h(e,Jf,n),h(e,Zo,n),a(Zo,Yb),h(e,Ef,n),h(e,C,n),a(C,Na),a(Na,Lb),a(Na,yn),a(yn,Db),a(Na,Xb),a(C,Vb),a(C,Qa),a(Qa,Hb),a(Qa,bn),a(bn,Ob),a(Qa,qb),a(C,Kb),a(C,Za),a(Za,eM),a(Za,Mn),a(Mn,tM),a(Za,aM),a(C,lM),a(C,vn),a(vn,rM),h(e,Af,n),h(e,Fo,n),a(Fo,oM),h(e,jf,n),h(e,Me,n),a(Me,xo),a(Me,sM),a(Me,Fa),a(Fa,iM),a(Me,nM),h(e,If,n),h(e,Yo,n),a(Yo,hM),h(e,kf,n),h(e,ve,n),a(ve,ot),a(ot,gn),u(xa,gn,null),a(ve,pM),a(ve,Tn),a(Tn,fM),h(e,$f,n),h(e,D,n),a(D,mM),a(D,Un),a(Un,cM),a(D,dM),a(D,_n),a(_n,uM),a(D,wM),h(e,zf,n),u(Ya,e,n),h(e,Cf,n),u(La,e,n),h(e,Gf,n),h(e,Lo,n),a(Lo,yM),h(e,Sf,n),u(Da,e,n),h(e,Rf,n),u(Xa,e,n),h(e,Pf,n),h(e,Do,n),a(Do,bM),h(e,Wf,n),h(e,ge,n),a(ge,st),a(st,Jn),u(Va,Jn,null),a(ge,MM),a(ge,En),a(En,vM),h(e,Bf,n),h(e,Xo,n),a(Xo,gM),h(e,Nf,n),u(Ha,e,n),h(e,Qf,n),h(e,Te,n),a(Te,it),a(it,An),u(Oa,An,null),a(Te,TM),a(Te,jn),a(jn,UM),h(e,Zf,n),h(e,Vo,n),a(Vo,_M),h(e,Ff,n),u(qa,e,n),h(e,xf,n),h(e,Ho,n),a(Ho,JM),h(e,Yf,n),h(e,nt,n),a(nt,EM),a(nt,Ka),a(Ka,AM),a(nt,jM),h(e,Lf,n),h(e,Oo,n),a(Oo,IM),h(e,Df,n),h(e,qo,n),a(qo,kM),h(e,Xf,n),u(el,e,n),h(e,Vf,n),h(e,Ko,n),a(Ko,$M),h(e,Hf,n),h(e,X,n),a(X,zM),a(X,In),a(In,CM),a(X,GM),a(X,kn),a(kn,SM),a(X,RM),h(e,Of,n),h(e,ht,n),a(ht,PM),a(ht,$n),a($n,WM),a(ht,BM),h(e,qf,n),h(e,V,n),a(V,NM),a(V,tl),a(tl,QM),a(V,ZM),a(V,al),a(al,FM),a(V,xM),h(e,Kf,n),h(e,es,n),a(es,YM),h(e,em,n),h(e,Ue,n),a(Ue,pt),a(pt,zn),u(ll,zn,null),a(Ue,LM),a(Ue,Cn),a(Cn,DM),h(e,tm,n),h(e,ts,n),a(ts,XM),h(e,am,n),h(e,ft,n),a(ft,VM),a(ft,Gn),a(Gn,HM),a(ft,OM),h(e,lm,n),h(e,as,n),a(as,qM),h(e,rm,n),u(rl,e,n),h(e,om,n),h(e,H,n),a(H,KM),a(H,ol),a(ol,ev),a(H,tv),a(H,Sn),a(Sn,av),a(H,lv),h(e,sm,n),h(e,mt,n),a(mt,rv),a(mt,sl),a(sl,ov),a(mt,sv),h(e,im,n),h(e,ct,n),a(ct,iv),a(ct,Rn),a(Rn,nv),a(ct,hv),h(e,nm,n),h(e,O,n),a(O,il),a(il,pv),a(il,Pn),a(Pn,fv),a(il,mv),a(O,cv),a(O,nl),a(nl,dv),a(nl,Wn),a(Wn,uv),a(nl,wv),a(O,yv),a(O,hl),a(hl,bv),a(hl,Bn),a(Bn,Mv),a(hl,vv),h(e,hm,n),h(e,ls,n),a(ls,gv),h(e,pm,n),h(e,_e,n),a(_e,dt),a(dt,Nn),u(pl,Nn,null),a(_e,Tv),a(_e,Qn),a(Qn,Uv),h(e,fm,n),h(e,ut,n),a(ut,_v),a(ut,Zn),a(Zn,Jv),a(ut,Ev),h(e,mm,n),u(fl,e,n),h(e,cm,n),u(ml,e,n),h(e,dm,n),h(e,rs,n),a(rs,Av),h(e,um,n),u(cl,e,n),h(e,wm,n),u(dl,e,n),h(e,ym,n),h(e,os,n),a(os,jv),h(e,bm,n),h(e,Je,n),a(Je,wt),a(wt,Fn),u(ul,Fn,null),a(Je,Iv),a(Je,xn),a(xn,kv),h(e,Mm,n),h(e,ss,n),a(ss,$v),h(e,vm,n),h(e,j,n),a(j,zv),a(j,is),a(is,Cv),a(j,Gv),a(j,ns),a(ns,Sv),a(j,Rv),a(j,wl),a(wl,Pv),a(j,Wv),a(j,Yn),a(Yn,Bv),a(j,Nv),h(e,gm,n),h(e,hs,n),a(hs,Qv),h(e,Tm,n),u(yt,e,n),h(e,Um,n),u(yl,e,n),h(e,_m,n),h(e,bt,n),a(bt,Zv),a(bt,Ln),a(Ln,Fv),a(bt,xv),h(e,Jm,n),u(bl,e,n),h(e,Em,n),u(Ml,e,n),h(e,Am,n),h(e,ps,n),a(ps,Yv),h(e,jm,n),u(vl,e,n),h(e,Im,n),u(gl,e,n),h(e,km,n),h(e,fs,n),a(fs,Lv),h(e,$m,n),h(e,ms,n),a(ms,cs),h(e,zm,n),h(e,Ee,n),a(Ee,Mt),a(Mt,Dn),u(Tl,Dn,null),a(Ee,Dv),a(Ee,Xn),a(Xn,Vn),a(Vn,Xv),h(e,Cm,n),h(e,Ae,n),a(Ae,vt),a(vt,Hn),u(Ul,Hn,null),a(Ae,Vv),a(Ae,On),a(On,Hv),h(e,Gm,n),h(e,q,n),a(q,Ov),a(q,ds),a(ds,qv),a(q,Kv),a(q,us),a(us,eg),a(q,tg),h(e,Sm,n),u(_l,e,n),h(e,Rm,n),h(e,ws,n),a(ws,ag),h(e,Pm,n),u(Jl,e,n),h(e,Wm,n),h(e,J,n),a(J,lg),a(J,El),a(El,qn),a(qn,rg),a(J,og),a(J,ys),a(ys,sg),a(J,ig),a(J,Al),a(Al,Kn),a(Kn,ng),a(J,hg),a(J,eh),a(eh,pg),a(J,fg),a(J,jl),a(jl,th),a(th,mg),a(J,cg),h(e,Bm,n),h(e,gt,n),a(gt,dg),a(gt,ah),a(ah,ug),a(gt,wg),h(e,Nm,n),u(Il,e,n),h(e,Qm,n),h(e,Tt,n),a(Tt,yg),a(Tt,kl),a(kl,bg),a(Tt,Mg),h(e,Zm,n),h(e,je,n),a(je,Ut),a(Ut,lh),u($l,lh,null),a(je,vg),a(je,rh),a(rh,gg),h(e,Fm,n),h(e,bs,n),a(bs,Tg),h(e,xm,n),h(e,_t,n),a(_t,Ms),a(Ms,oh),a(oh,Ug),a(Ms,_g),a(_t,Jg),a(_t,vs),a(vs,sh),a(sh,Eg),a(vs,Ag),h(e,Ym,n),h(e,Ie,n),a(Ie,Jt),a(Jt,ih),u(zl,ih,null),a(Ie,jg),a(Ie,nh),a(nh,Ig),h(e,Lm,n),h(e,Et,n),a(Et,kg),a(Et,gs),a(gs,$g),a(Et,zg),h(e,Dm,n),h(e,Ts,n),a(Ts,Cg),h(e,Xm,n),h(e,At,n),a(At,hh),a(hh,Gg),a(At,Sg),a(At,ph),a(ph,Rg),h(e,Vm,n),h(e,jt,n),a(jt,Pg),a(jt,Us),a(Us,Wg),a(jt,Bg),h(e,Hm,n),h(e,_s,n),a(_s,Ng),h(e,Om,n),h(e,G,n),a(G,fh),a(fh,Cl),a(Cl,Qg),a(Cl,Js),a(Js,Zg),a(Cl,Fg),a(G,xg),a(G,Gl),a(Gl,mh),a(mh,Yg),a(Gl,Lg),u(Sl,Gl,null),a(G,Dg),a(G,ch),a(ch,Rl),a(Rl,Xg),a(Rl,Es),a(Es,Vg),a(Rl,Hg),a(G,Og),a(G,dh),a(dh,ke),a(ke,qg),a(ke,Pl),a(Pl,Kg),a(ke,e1),a(ke,uh),a(uh,t1),a(ke,a1),h(e,qm,n),h(e,$e,n),a($e,It),a(It,wh),u(Wl,wh,null),a($e,l1),a($e,yh),a(yh,r1),h(e,Km,n),h(e,As,n),a(As,o1),h(e,ec,n),h(e,ze,n),a(ze,kt),a(kt,bh),u(Bl,bh,null),a(ze,s1),a(ze,Mh),a(Mh,i1),h(e,tc,n),h(e,js,n),a(js,n1),h(e,ac,n),h(e,$t,n),a($t,vh),a(vh,h1),a($t,p1),a($t,gh),a(gh,f1),h(e,lc,n),h(e,Is,n),a(Is,m1),h(e,rc,n),h(e,ks,n),a(ks,c1),h(e,oc,n),h(e,Ce,n),a(Ce,zt),a(zt,Th),u(Nl,Th,null),a(Ce,d1),a(Ce,Uh),a(Uh,u1),h(e,sc,n),h(e,Ct,n),a(Ct,w1),a(Ct,Ql),a(Ql,y1),a(Ct,b1),h(e,ic,n),h(e,Gt,n),a(Gt,M1),a(Gt,_h),a(_h,v1),a(Gt,g1),h(e,nc,n),h(e,$s,n),a($s,T1),h(e,hc,n),h(e,K,n),a(K,U1),a(K,Zl),a(Zl,_1),a(K,J1),a(K,Fl),a(Fl,E1),a(K,A1),h(e,pc,n),h(e,zs,n),a(zs,j1),h(e,fc,n),h(e,Ge,n),a(Ge,St),a(St,Jh),u(xl,Jh,null),a(Ge,I1),a(Ge,Eh),a(Eh,k1),h(e,mc,n),h(e,Se,n),a(Se,Rt),a(Rt,Ah),u(Yl,Ah,null),a(Se,$1),a(Se,jh),a(jh,z1),h(e,cc,n),h(e,Cs,n),a(Cs,C1),h(e,dc,n),h(e,Gs,n),a(Gs,G1),h(e,uc,n),h(e,Ss,n),a(Ss,S1),h(e,wc,n),h(e,Rs,n),a(Rs,Ps),h(e,yc,n),h(e,Pt,n),a(Pt,R1),a(Pt,Ll),a(Ll,P1),a(Pt,W1),h(e,bc,n),h(e,Ws,n),a(Ws,B1),h(e,Mc,n),h(e,Bs,n),a(Bs,N1),h(e,vc,n),h(e,Ns,n),a(Ns,Q1),h(e,gc,n),h(e,Qs,n),a(Qs,Z1),h(e,Tc,n),h(e,ee,n),a(ee,Ih),a(Ih,Dl),a(Dl,F1),a(ee,x1),a(ee,kh),a(kh,Xl),a(Xl,Y1),a(ee,L1),a(ee,$h),a($h,Vl),a(Vl,D1),h(e,Uc,n),h(e,T,n),a(T,X1),a(T,Hl),a(Hl,V1),a(T,H1),a(T,Ol),a(Ol,O1),a(T,q1),a(T,ql),a(ql,K1),a(T,eT),a(T,Kl),a(Kl,tT),a(T,aT),a(T,er),a(er,lT),a(T,rT),a(T,Zs),a(Zs,oT),a(T,sT),h(e,_c,n),h(e,Re,n),a(Re,Wt),a(Wt,zh),u(tr,zh,null),a(Re,iT),a(Re,Ch),a(Ch,nT),h(e,Jc,n),h(e,Fs,n),a(Fs,hT),h(e,Ec,n),h(e,Bt,n),a(Bt,pT),a(Bt,xs),a(xs,fT),a(Bt,mT),h(e,Ac,n),h(e,Pe,n),a(Pe,Nt),a(Nt,Gh),u(ar,Gh,null),a(Pe,cT),a(Pe,Sh),a(Sh,dT),h(e,jc,n),h(e,S,n),a(S,uT),a(S,lr),a(lr,Rh),a(Rh,wT),a(S,yT),a(S,rr),a(rr,bT),a(S,MT),a(S,or),a(or,vT),a(S,gT),h(e,Ic,n),h(e,Qt,n),a(Qt,TT),a(Qt,sr),a(sr,Ph),a(Ph,UT),a(Qt,_T),h(e,kc,n),u(ir,e,n),h(e,$c,n),h(e,Ys,n),a(Ys,JT),h(e,zc,n),h(e,We,n),a(We,Zt),a(Zt,Wh),u(nr,Wh,null),a(We,ET),a(We,Bh),a(Bh,AT),h(e,Cc,n),h(e,Ft,n),a(Ft,jT),a(Ft,hr),a(hr,IT),a(Ft,kT),h(e,Gc,n),h(e,B,n),a(B,Nh),a(Nh,$T),a(B,zT),a(B,pr),a(pr,CT),a(B,GT),a(B,Qh),a(Qh,ST),a(B,RT),h(e,Sc,n),h(e,Ls,n),a(Ls,PT),h(e,Rc,n),h(e,fr,n),a(fr,Zh),a(Zh,WT),a(fr,BT),h(e,Pc,n),h(e,xt,n),a(xt,Ds),a(Ds,Fh),a(Fh,NT),a(Ds,QT),a(xt,ZT),a(xt,Xs),a(Xs,xh),a(xh,FT),a(Xs,xT),h(e,Wc,n),h(e,mr,n),a(mr,Yh),a(Yh,YT),a(mr,LT),h(e,Bc,n),h(e,R,n),a(R,cr),a(cr,Lh),a(Lh,DT),a(cr,XT),a(cr,dr),a(dr,VT),a(R,HT),a(R,ur),a(ur,Dh),a(Dh,OT),a(ur,qT),a(ur,wr),a(wr,KT),a(R,e2),a(R,yr),a(yr,Xh),a(Xh,t2),a(yr,a2),a(yr,br),a(br,l2),a(R,r2),a(R,Mr),a(Mr,Vh),a(Vh,o2),a(Mr,s2),a(Mr,vr),a(vr,i2),h(e,Nc,n),h(e,gr,n),a(gr,Hh),a(Hh,n2),a(gr,h2),h(e,Qc,n),h(e,P,n),a(P,Tr),a(Tr,Oh),a(Oh,p2),a(Tr,f2),a(Tr,Ur),a(Ur,m2),a(P,c2),a(P,_r),a(_r,qh),a(qh,d2),a(_r,u2),a(_r,Jr),a(Jr,w2),a(P,y2),a(P,Er),a(Er,Kh),a(Kh,b2),a(Er,M2),a(Er,Ar),a(Ar,v2),a(P,g2),a(P,jr),a(jr,ep),a(ep,T2),a(jr,U2),a(jr,Ir),a(Ir,_2),Zc=!0},p(e,[n]){const kr={};n&2&&(kr.$$scope={dirty:n,ctx:e}),Le.$set(kr);const tp={};n&2&&(tp.$$scope={dirty:n,ctx:e}),yt.$set(tp)},i(e){Zc||(w(Ot.$$.fragment,e),w(Kt.$$.fragment,e),w(ta.$$.fragment,e),w(aa.$$.fragment,e),w(la.$$.fragment,e),w(ra.$$.fragment,e),w(oa.$$.fragment,e),w(sa.$$.fragment,e),w(ia.$$.fragment,e),w(na.$$.fragment,e),w(ha.$$.fragment,e),w(Le.$$.fragment,e),w(pa.$$.fragment,e),w(fa.$$.fragment,e),w(ma.$$.fragment,e),w(ca.$$.fragment,e),w(Ta.$$.fragment,e),w(_a.$$.fragment,e),w(ka.$$.fragment,e),w($a.$$.fragment,e),w(za.$$.fragment,e),w(Sa.$$.fragment,e),w(Pa.$$.fragment,e),w(Wa.$$.fragment,e),w(Ba.$$.fragment,e),w(xa.$$.fragment,e),w(Ya.$$.fragment,e),w(La.$$.fragment,e),w(Da.$$.fragment,e),w(Xa.$$.fragment,e),w(Va.$$.fragment,e),w(Ha.$$.fragment,e),w(Oa.$$.fragment,e),w(qa.$$.fragment,e),w(el.$$.fragment,e),w(ll.$$.fragment,e),w(rl.$$.fragment,e),w(pl.$$.fragment,e),w(fl.$$.fragment,e),w(ml.$$.fragment,e),w(cl.$$.fragment,e),w(dl.$$.fragment,e),w(ul.$$.fragment,e),w(yt.$$.fragment,e),w(yl.$$.fragment,e),w(bl.$$.fragment,e),w(Ml.$$.fragment,e),w(vl.$$.fragment,e),w(gl.$$.fragment,e),w(Tl.$$.fragment,e),w(Ul.$$.fragment,e),w(_l.$$.fragment,e),w(Jl.$$.fragment,e),w(Il.$$.fragment,e),w($l.$$.fragment,e),w(zl.$$.fragment,e),w(Sl.$$.fragment,e),w(Wl.$$.fragment,e),w(Bl.$$.fragment,e),w(Nl.$$.fragment,e),w(xl.$$.fragment,e),w(Yl.$$.fragment,e),w(tr.$$.fragment,e),w(ar.$$.fragment,e),w(ir.$$.fragment,e),w(nr.$$.fragment,e),Zc=!0)},o(e){y(Ot.$$.fragment,e),y(Kt.$$.fragment,e),y(ta.$$.fragment,e),y(aa.$$.fragment,e),y(la.$$.fragment,e),y(ra.$$.fragment,e),y(oa.$$.fragment,e),y(sa.$$.fragment,e),y(ia.$$.fragment,e),y(na.$$.fragment,e),y(ha.$$.fragment,e),y(Le.$$.fragment,e),y(pa.$$.fragment,e),y(fa.$$.fragment,e),y(ma.$$.fragment,e),y(ca.$$.fragment,e),y(Ta.$$.fragment,e),y(_a.$$.fragment,e),y(ka.$$.fragment,e),y($a.$$.fragment,e),y(za.$$.fragment,e),y(Sa.$$.fragment,e),y(Pa.$$.fragment,e),y(Wa.$$.fragment,e),y(Ba.$$.fragment,e),y(xa.$$.fragment,e),y(Ya.$$.fragment,e),y(La.$$.fragment,e),y(Da.$$.fragment,e),y(Xa.$$.fragment,e),y(Va.$$.fragment,e),y(Ha.$$.fragment,e),y(Oa.$$.fragment,e),y(qa.$$.fragment,e),y(el.$$.fragment,e),y(ll.$$.fragment,e),y(rl.$$.fragment,e),y(pl.$$.fragment,e),y(fl.$$.fragment,e),y(ml.$$.fragment,e),y(cl.$$.fragment,e),y(dl.$$.fragment,e),y(ul.$$.fragment,e),y(yt.$$.fragment,e),y(yl.$$.fragment,e),y(bl.$$.fragment,e),y(Ml.$$.fragment,e),y(vl.$$.fragment,e),y(gl.$$.fragment,e),y(Tl.$$.fragment,e),y(Ul.$$.fragment,e),y(_l.$$.fragment,e),y(Jl.$$.fragment,e),y(Il.$$.fragment,e),y($l.$$.fragment,e),y(zl.$$.fragment,e),y(Sl.$$.fragment,e),y(Wl.$$.fragment,e),y(Bl.$$.fragment,e),y(Nl.$$.fragment,e),y(xl.$$.fragment,e),y(Yl.$$.fragment,e),y(tr.$$.fragment,e),y(ar.$$.fragment,e),y(ir.$$.fragment,e),y(nr.$$.fragment,e),Zc=!1},d(e){t(g),e&&t(E),e&&t(U),b(Ot),e&&t(rp),e&&t(Be),e&&t(op),e&&t(N),e&&t(sp),e&&t(Ne),e&&t(ip),e&&t(oo),e&&t(np),b(Kt,e),e&&t(hp),e&&t(Q),e&&t(pp),e&&t(Qe),e&&t(fp),b(ta,e),e&&t(mp),e&&t(Ze),e&&t(cp),b(aa,e),e&&t(dp),e&&t(io),e&&t(up),b(la,e),e&&t(wp),e&&t(no),e&&t(yp),b(ra,e),e&&t(bp),e&&t(ho),e&&t(Mp),e&&t(fe),b(oa),e&&t(vp),e&&t(xe),e&&t(gp),b(sa,e),e&&t(Tp),e&&t(Ye),e&&t(Up),b(ia,e),e&&t(_p),b(na,e),e&&t(Jp),e&&t(po),e&&t(Ep),b(ha,e),e&&t(Ap),b(Le,e),e&&t(jp),e&&t(me),b(pa),e&&t(Ip),e&&t(Xe),e&&t(kp),b(fa,e),e&&t($p),b(ma,e),e&&t(zp),e&&t(mo),e&&t(Cp),e&&t(ce),b(ca),e&&t(Gp),e&&t(co),e&&t(Sp),e&&t(Z),e&&t(Rp),e&&t(uo),e&&t(Pp),e&&t(va),e&&t(Wp),e&&t(de),b(Ta),e&&t(Bp),e&&t(wo),e&&t(Np),e&&t(yo),e&&t(Qp),e&&t(bo),e&&t(Zp),e&&t(Oe),e&&t(Fp),e&&t(Mo),e&&t(xp),e&&t(F),e&&t(Yp),e&&t(go),e&&t(Lp),e&&t(To),e&&t(Dp),e&&t(Uo),e&&t(Xp),e&&t(_o),e&&t(Vp),e&&t(Jo),e&&t(Hp),e&&t(Eo),e&&t(Op),e&&t(Ao),e&&t(qp),e&&t(jo),e&&t(Kp),e&&t(Io),e&&t(ef),e&&t(ko),e&&t(tf),e&&t($o),e&&t(af),e&&t(zo),e&&t(lf),e&&t(ue),b(_a),e&&t(rf),e&&t(Co),e&&t(of),e&&t(x),e&&t(sf),e&&t(Aa),e&&t(nf),e&&t(et),e&&t(hf),e&&t(we),b(ka),e&&t(pf),e&&t(Go),e&&t(ff),e&&t(z),e&&t(mf),b($a,e),e&&t(cf),b(za,e),e&&t(df),e&&t(A),e&&t(uf),e&&t(Y),e&&t(wf),e&&t(Po),e&&t(yf),e&&t(ye),b(Sa),e&&t(bf),e&&t(Wo),e&&t(Mf),e&&t(lt),e&&t(vf),e&&t(L),e&&t(gf),b(Pa,e),e&&t(Tf),b(Wa,e),e&&t(Uf),e&&t(Qo),e&&t(_f),e&&t(be),b(Ba),e&&t(Jf),e&&t(Zo),e&&t(Ef),e&&t(C),e&&t(Af),e&&t(Fo),e&&t(jf),e&&t(Me),e&&t(If),e&&t(Yo),e&&t(kf),e&&t(ve),b(xa),e&&t($f),e&&t(D),e&&t(zf),b(Ya,e),e&&t(Cf),b(La,e),e&&t(Gf),e&&t(Lo),e&&t(Sf),b(Da,e),e&&t(Rf),b(Xa,e),e&&t(Pf),e&&t(Do),e&&t(Wf),e&&t(ge),b(Va),e&&t(Bf),e&&t(Xo),e&&t(Nf),b(Ha,e),e&&t(Qf),e&&t(Te),b(Oa),e&&t(Zf),e&&t(Vo),e&&t(Ff),b(qa,e),e&&t(xf),e&&t(Ho),e&&t(Yf),e&&t(nt),e&&t(Lf),e&&t(Oo),e&&t(Df),e&&t(qo),e&&t(Xf),b(el,e),e&&t(Vf),e&&t(Ko),e&&t(Hf),e&&t(X),e&&t(Of),e&&t(ht),e&&t(qf),e&&t(V),e&&t(Kf),e&&t(es),e&&t(em),e&&t(Ue),b(ll),e&&t(tm),e&&t(ts),e&&t(am),e&&t(ft),e&&t(lm),e&&t(as),e&&t(rm),b(rl,e),e&&t(om),e&&t(H),e&&t(sm),e&&t(mt),e&&t(im),e&&t(ct),e&&t(nm),e&&t(O),e&&t(hm),e&&t(ls),e&&t(pm),e&&t(_e),b(pl),e&&t(fm),e&&t(ut),e&&t(mm),b(fl,e),e&&t(cm),b(ml,e),e&&t(dm),e&&t(rs),e&&t(um),b(cl,e),e&&t(wm),b(dl,e),e&&t(ym),e&&t(os),e&&t(bm),e&&t(Je),b(ul),e&&t(Mm),e&&t(ss),e&&t(vm),e&&t(j),e&&t(gm),e&&t(hs),e&&t(Tm),b(yt,e),e&&t(Um),b(yl,e),e&&t(_m),e&&t(bt),e&&t(Jm),b(bl,e),e&&t(Em),b(Ml,e),e&&t(Am),e&&t(ps),e&&t(jm),b(vl,e),e&&t(Im),b(gl,e),e&&t(km),e&&t(fs),e&&t($m),e&&t(ms),e&&t(zm),e&&t(Ee),b(Tl),e&&t(Cm),e&&t(Ae),b(Ul),e&&t(Gm),e&&t(q),e&&t(Sm),b(_l,e),e&&t(Rm),e&&t(ws),e&&t(Pm),b(Jl,e),e&&t(Wm),e&&t(J),e&&t(Bm),e&&t(gt),e&&t(Nm),b(Il,e),e&&t(Qm),e&&t(Tt),e&&t(Zm),e&&t(je),b($l),e&&t(Fm),e&&t(bs),e&&t(xm),e&&t(_t),e&&t(Ym),e&&t(Ie),b(zl),e&&t(Lm),e&&t(Et),e&&t(Dm),e&&t(Ts),e&&t(Xm),e&&t(At),e&&t(Vm),e&&t(jt),e&&t(Hm),e&&t(_s),e&&t(Om),e&&t(G),b(Sl),e&&t(qm),e&&t($e),b(Wl),e&&t(Km),e&&t(As),e&&t(ec),e&&t(ze),b(Bl),e&&t(tc),e&&t(js),e&&t(ac),e&&t($t),e&&t(lc),e&&t(Is),e&&t(rc),e&&t(ks),e&&t(oc),e&&t(Ce),b(Nl),e&&t(sc),e&&t(Ct),e&&t(ic),e&&t(Gt),e&&t(nc),e&&t($s),e&&t(hc),e&&t(K),e&&t(pc),e&&t(zs),e&&t(fc),e&&t(Ge),b(xl),e&&t(mc),e&&t(Se),b(Yl),e&&t(cc),e&&t(Cs),e&&t(dc),e&&t(Gs),e&&t(uc),e&&t(Ss),e&&t(wc),e&&t(Rs),e&&t(yc),e&&t(Pt),e&&t(bc),e&&t(Ws),e&&t(Mc),e&&t(Bs),e&&t(vc),e&&t(Ns),e&&t(gc),e&&t(Qs),e&&t(Tc),e&&t(ee),e&&t(Uc),e&&t(T),e&&t(_c),e&&t(Re),b(tr),e&&t(Jc),e&&t(Fs),e&&t(Ec),e&&t(Bt),e&&t(Ac),e&&t(Pe),b(ar),e&&t(jc),e&&t(S),e&&t(Ic),e&&t(Qt),e&&t(kc),b(ir,e),e&&t($c),e&&t(Ys),e&&t(zc),e&&t(We),b(nr),e&&t(Cc),e&&t(Ft),e&&t(Gc),e&&t(B),e&&t(Sc),e&&t(Ls),e&&t(Rc),e&&t(fr),e&&t(Pc),e&&t(xt),e&&t(Wc),e&&t(mr),e&&t(Bc),e&&t(R),e&&t(Nc),e&&t(gr),e&&t(Qc),e&&t(P)}}}const lA={local:"efficient-training-on-a-single-gpu",sections:[{local:"load-model",title:"Load Model"},{local:"vanilla-training",title:"Vanilla Training"},{local:"anatomy-of-models-operations",title:"Anatomy of Model's Operations"},{local:"anatomy-of-models-memory",title:"Anatomy of Model's Memory"},{local:"batch-sizes",title:"Batch sizes"},{local:"gradient-accumulation",title:"Gradient Accumulation"},{local:"gradient-checkpointing",title:"Gradient Checkpointing"},{local:"floating-data-types",sections:[{local:"fp16-training",title:"FP16 Training"},{local:"bf16",title:"BF16"},{local:"tf32",title:"TF32"}],title:"Floating Data Types"},{local:"optimizer",sections:[{local:"adafactor",title:"Adafactor"},{local:"8bit-adam",title:"8-bit Adam"},{local:"multitensor",title:"`_multi_tensor`"}],title:"Optimizer"},{local:"using-accelerate",title:"Using \u{1F917} Accelerate"},{local:"dataloader",title:"DataLoader"},{local:"deepspeed-zero",title:"DeepSpeed ZeRO"},{local:"choice-of-gpu",title:"Choice of GPU"},{local:"how-to-scale",title:"How to scale"},{local:"efficient-software-prebuilds",title:"Efficient Software Prebuilds"},{local:"sparsity",sections:[{local:"mixture-of-experts",title:"Mixture of Experts"}],title:"Sparsity"},{local:"scaling-beyond-a-single-gpu",title:"Scaling beyond a single GPU"},{local:"using-pytorch-native-attention",title:"Using PyTorch native attention"},{local:"using-torchcompile",title:"Using torch.compile"}],title:"Efficient Training on a Single GPU"};function rA(zr){return K3(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class hA extends V3{constructor(g){super();H3(this,g,rA,aA,O3,{})}}export{hA as default,lA as metadata};
