import{S as Xl,i as Vl,s as Kl,e as r,k as p,w as _,t as s,M as Zl,c as a,d as o,m as u,a as l,x as v,h as i,b as m,N as Io,G as t,g as f,y as g,q as w,o as y,B as $,v as Jl}from"../chunks/vendor-hf-doc-builder.js";import{T as Ql}from"../chunks/Tip-hf-doc-builder.js";import{I as xe}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as tt}from"../chunks/CodeBlock-hf-doc-builder.js";function Yl(Dt){let d,S;return{c(){d=r("p"),S=s("Note that this feature can also be used in a multi GPU setup.")},l(c){d=a(c,"P",{});var E=l(d);S=i(E,"Note that this feature can also be used in a multi GPU setup."),E.forEach(o)},m(c,E){f(c,d,E),t(d,S)},d(c){c&&o(d)}}}function es(Dt){let d,S,c,E,ot,ee,Oo,rt,Lo,St,A,Bo,Ae,jo,Do,ke,So,Fo,Ft,U,F,at,te,No,lt,Ho,Nt,k,zo,oe,st,Ro,Wo,re,Xo,Vo,Ht,T,Ko,ae,Zo,Jo,le,Qo,Yo,zt,G,er,se,it,tr,or,Te,rr,ar,Rt,ie,Wt,N,lr,Ge,sr,ir,Xt,ne,Vt,H,nr,fe,fr,mr,Kt,I,z,nt,me,pr,Me,ft,ur,dr,Zt,R,Jt,h,hr,pe,mt,cr,br,pt,_r,vr,ut,gr,wr,dt,yr,$r,ht,Er,Pr,Qt,Ce,qe,Ta,Yt,M,xr,ue,Ar,kr,de,Tr,Gr,eo,Ue,Ie,Ga,to,W,Mr,Oe,Cr,qr,oo,O,X,ct,he,Ur,bt,Ir,ro,C,L,Or,_t,Lr,Br,vt,jr,Dr,Sr,V,Fr,gt,Nr,Hr,wt,zr,Rr,K,Wr,yt,Xr,Vr,$t,Kr,ao,B,Z,Et,ce,Zr,Pt,Jr,lo,Le,Qr,so,be,io,Be,Yr,no,J,b,ea,xt,ta,oa,At,ra,aa,kt,la,sa,Tt,ia,na,Gt,fa,ma,pa,Mt,ua,fo,je,da,mo,_e,po,j,Q,Ct,ve,ha,qt,ca,uo,De,ba,ho,ge,co,q,_a,Ut,va,ga,It,wa,ya,bo,we,_o,Se,$a,vo,D,Y,Ot,ye,Ea,Lt,Pa,go,Fe,xa,wo,Ne,$e,He,Ma,yo,ze,Aa,$o,Re,Ee,We,Ca,Eo;return ee=new xe({}),te=new xe({}),ie=new tt({props:{code:"model = model.to_bettertransformer()",highlighted:"model = model.to_bettertransformer()"}}),ne=new tt({props:{code:`model = model.reverse_bettertransformer()
model.save_pretrained("saved_model")`,highlighted:`model = model.reverse_bettertransformer()
model.save_pretrained(<span class="hljs-string">&quot;saved_model&quot;</span>)`}}),me=new xe({}),R=new Ql({props:{$$slots:{default:[Yl]},$$scope:{ctx:Dt}}}),he=new xe({}),ce=new xe({}),be=new tt({props:{code:`from transformers import AutoModelForCausalLM

model_name = "bigscience/bloom-2b5"
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),_e=new tt({props:{code:`from transformers import AutoModelForCausalLM, AutoTokenizer

model_name = "bigscience/bloom-2b5"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True)

prompt = "Hello, my llama is cute"
inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
generated_ids = model.generate(**inputs)
outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_name)
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)

prompt = <span class="hljs-string">&quot;Hello, my llama is cute&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
generated_ids = model.generate(**inputs)
outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)`}}),ve=new xe({}),ge=new tt({props:{code:`model_name = "bigscience/bloom-2b5"
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True)`,highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),we=new tt({props:{code:`max_memory_mapping = {0: "1GB", 1: "2GB"}
model_name = "bigscience/bloom-3b"
model_8bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map="auto", load_in_8bit=True, max_memory=max_memory_mapping
)`,highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;1GB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;2GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`}}),ye=new xe({}),{c(){d=r("meta"),S=p(),c=r("h1"),E=r("a"),ot=r("span"),_(ee.$$.fragment),Oo=p(),rt=r("span"),Lo=s("Efficient Inference on a Single GPU"),St=p(),A=r("p"),Bo=s("In addition to this guide, relevant information can be found as well in "),Ae=r("a"),jo=s("the guide for training on a single GPU"),Do=s(" and "),ke=r("a"),So=s("the guide for inference on CPUs"),Fo=s("."),Ft=p(),U=r("h2"),F=r("a"),at=r("span"),_(te.$$.fragment),No=p(),lt=r("span"),Ho=s("Better Transformer: PyTorch-native transformer fastpath"),Nt=p(),k=r("p"),zo=s("PyTorch-native "),oe=r("a"),st=r("code"),Ro=s("nn.MultiHeadAttention"),Wo=s(" attention fastpath, called BetterTransformer, can be used with Transformers through the integration in the "),re=r("a"),Xo=s("\u{1F917} Optimum library"),Vo=s("."),Ht=p(),T=r("p"),Ko=s("PyTorch\u2019s attention fastpath allows to speed up inference through kernel fusions and the use of "),ae=r("a"),Zo=s("nested tensors"),Jo=s(". Detailed benchmarks can be found in "),le=r("a"),Qo=s("this blog post"),Yo=s("."),zt=p(),G=r("p"),er=s("After installing the "),se=r("a"),it=r("code"),tr=s("optimum"),or=s(" package, to use Better Transformer during inference, the relevant internal modules are replaced by calling "),Te=r("a"),rr=s("to_bettertransformer()"),ar=s(":"),Rt=p(),_(ie.$$.fragment),Wt=p(),N=r("p"),lr=s("The method "),Ge=r("a"),sr=s("reverse_bettertransformer()"),ir=s(" allows to go back to the original modeling, which should be used before saving the model in order to use the canonical transformers modeling:"),Xt=p(),_(ne.$$.fragment),Vt=p(),H=r("p"),nr=s("As of PyTorch 2.0, the attention fastpath is supported for both encoders and decoders. The list of supported architectures can be found "),fe=r("a"),fr=s("here"),mr=s("."),Kt=p(),I=r("h2"),z=r("a"),nt=r("span"),_(me.$$.fragment),pr=p(),Me=r("span"),ft=r("code"),ur=s("bitsandbytes"),dr=s(" integration for Int8 mixed-precision matrix decomposition"),Zt=p(),_(R.$$.fragment),Jt=p(),h=r("p"),hr=s("From the paper "),pe=r("a"),mt=r("code"),cr=s("LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),br=s(`, we support Hugging Face integration for all models in the Hub with a few lines of code.
The method reduces `),pt=r("code"),_r=s("nn.Linear"),vr=s(" size by 2 for "),ut=r("code"),gr=s("float16"),wr=s(" and "),dt=r("code"),yr=s("bfloat16"),$r=s(" weights and by 4 for "),ht=r("code"),Er=s("float32"),Pr=s(" weights, with close to no impact to the quality by operating on the outliers in half-precision."),Qt=p(),Ce=r("p"),qe=r("img"),Yt=p(),M=r("p"),xr=s(`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),ue=r("a"),Ar=s("paper"),kr=s(" or our "),de=r("a"),Tr=s("blogpost about the integration"),Gr=s("."),eo=p(),Ue=r("p"),Ie=r("img"),to=p(),W=r("p"),Mr=s(`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),Oe=r("a"),Cr=s("Google colab"),qr=s("."),oo=p(),O=r("h3"),X=r("a"),ct=r("span"),_(he.$$.fragment),Ur=p(),bt=r("span"),Ir=s("Requirements"),ro=p(),C=r("ul"),L=r("li"),Or=s("If you have "),_t=r("code"),Lr=s("bitsandbytes<0.37.0"),Br=s(", make sure you run on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100). For "),vt=r("code"),jr=s("bitsandbytes>=0.37.0"),Dr=s(", all GPUs should be supported."),Sr=p(),V=r("li"),Fr=s("Install the correct version of "),gt=r("code"),Nr=s("bitsandbytes"),Hr=s(` by running:
`),wt=r("code"),zr=s("pip install bitsandbytes>=0.31.5"),Rr=p(),K=r("li"),Wr=s("Install "),yt=r("code"),Xr=s("accelerate"),Vr=p(),$t=r("code"),Kr=s("pip install accelerate>=0.12.0"),ao=p(),B=r("h3"),Z=r("a"),Et=r("span"),_(ce.$$.fragment),Zr=p(),Pt=r("span"),Jr=s("Running mixed-Int8 models - single GPU setup"),lo=p(),Le=r("p"),Qr=s("After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),so=p(),_(be.$$.fragment),io=p(),Be=r("p"),Yr=s("For text generation, we recommend:"),no=p(),J=r("ul"),b=r("li"),ea=s("using the model\u2019s "),xt=r("code"),ta=s("generate()"),oa=s(" method instead of the "),At=r("code"),ra=s("pipeline()"),aa=s(" function. Although inference is possible with the "),kt=r("code"),la=s("pipeline()"),sa=s(" function, it is not optimized for mixed-8bit models, and will be slower than using the "),Tt=r("code"),ia=s("generate()"),na=s(" method. Moreover, some sampling strategies are like nucleaus sampling are not supported by the "),Gt=r("code"),fa=s("pipeline()"),ma=s(" function for mixed-8bit models."),pa=p(),Mt=r("li"),ua=s("placing all inputs on the same device as the model."),fo=p(),je=r("p"),da=s("Here is a simple example:"),mo=p(),_(_e.$$.fragment),po=p(),j=r("h3"),Q=r("a"),Ct=r("span"),_(ve.$$.fragment),ha=p(),qt=r("span"),ca=s("Running mixed-int8 models - multi GPU setup"),uo=p(),De=r("p"),ba=s("The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),ho=p(),_(ge.$$.fragment),co=p(),q=r("p"),_a=s("But you can control the GPU RAM you want to allocate on each GPU using "),Ut=r("code"),va=s("accelerate"),ga=s(". Use the "),It=r("code"),wa=s("max_memory"),ya=s(" argument as follows:"),bo=p(),_(we.$$.fragment),_o=p(),Se=r("p"),$a=s("In this example, the first GPU will use 1GB of memory and the second 2GB."),vo=p(),D=r("h3"),Y=r("a"),Ot=r("span"),_(ye.$$.fragment),Ea=p(),Lt=r("span"),Pa=s("Colab demos"),go=p(),Fe=r("p"),xa=s(`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),wo=p(),Ne=r("p"),$e=r("a"),He=r("img"),yo=p(),ze=r("p"),Aa=s("Or this demo for BLOOM-3B:"),$o=p(),Re=r("p"),Ee=r("a"),We=r("img"),this.h()},l(e){const n=Zl('[data-svelte="svelte-1phssyn"]',document.head);d=a(n,"META",{name:!0,content:!0}),n.forEach(o),S=u(e),c=a(e,"H1",{class:!0});var Pe=l(c);E=a(Pe,"A",{id:!0,class:!0,href:!0});var qa=l(E);ot=a(qa,"SPAN",{});var Ua=l(ot);v(ee.$$.fragment,Ua),Ua.forEach(o),qa.forEach(o),Oo=u(Pe),rt=a(Pe,"SPAN",{});var Ia=l(rt);Lo=i(Ia,"Efficient Inference on a Single GPU"),Ia.forEach(o),Pe.forEach(o),St=u(e),A=a(e,"P",{});var Xe=l(A);Bo=i(Xe,"In addition to this guide, relevant information can be found as well in "),Ae=a(Xe,"A",{href:!0});var Oa=l(Ae);jo=i(Oa,"the guide for training on a single GPU"),Oa.forEach(o),Do=i(Xe," and "),ke=a(Xe,"A",{href:!0});var La=l(ke);So=i(La,"the guide for inference on CPUs"),La.forEach(o),Fo=i(Xe,"."),Xe.forEach(o),Ft=u(e),U=a(e,"H2",{class:!0});var Po=l(U);F=a(Po,"A",{id:!0,class:!0,href:!0});var Ba=l(F);at=a(Ba,"SPAN",{});var ja=l(at);v(te.$$.fragment,ja),ja.forEach(o),Ba.forEach(o),No=u(Po),lt=a(Po,"SPAN",{});var Da=l(lt);Ho=i(Da,"Better Transformer: PyTorch-native transformer fastpath"),Da.forEach(o),Po.forEach(o),Nt=u(e),k=a(e,"P",{});var Ve=l(k);zo=i(Ve,"PyTorch-native "),oe=a(Ve,"A",{href:!0,rel:!0});var Sa=l(oe);st=a(Sa,"CODE",{});var Fa=l(st);Ro=i(Fa,"nn.MultiHeadAttention"),Fa.forEach(o),Sa.forEach(o),Wo=i(Ve," attention fastpath, called BetterTransformer, can be used with Transformers through the integration in the "),re=a(Ve,"A",{href:!0,rel:!0});var Na=l(re);Xo=i(Na,"\u{1F917} Optimum library"),Na.forEach(o),Vo=i(Ve,"."),Ve.forEach(o),Ht=u(e),T=a(e,"P",{});var Ke=l(T);Ko=i(Ke,"PyTorch\u2019s attention fastpath allows to speed up inference through kernel fusions and the use of "),ae=a(Ke,"A",{href:!0,rel:!0});var Ha=l(ae);Zo=i(Ha,"nested tensors"),Ha.forEach(o),Jo=i(Ke,". Detailed benchmarks can be found in "),le=a(Ke,"A",{href:!0,rel:!0});var za=l(le);Qo=i(za,"this blog post"),za.forEach(o),Yo=i(Ke,"."),Ke.forEach(o),zt=u(e),G=a(e,"P",{});var Ze=l(G);er=i(Ze,"After installing the "),se=a(Ze,"A",{href:!0,rel:!0});var Ra=l(se);it=a(Ra,"CODE",{});var Wa=l(it);tr=i(Wa,"optimum"),Wa.forEach(o),Ra.forEach(o),or=i(Ze," package, to use Better Transformer during inference, the relevant internal modules are replaced by calling "),Te=a(Ze,"A",{href:!0});var Xa=l(Te);rr=i(Xa,"to_bettertransformer()"),Xa.forEach(o),ar=i(Ze,":"),Ze.forEach(o),Rt=u(e),v(ie.$$.fragment,e),Wt=u(e),N=a(e,"P",{});var xo=l(N);lr=i(xo,"The method "),Ge=a(xo,"A",{href:!0});var Va=l(Ge);sr=i(Va,"reverse_bettertransformer()"),Va.forEach(o),ir=i(xo," allows to go back to the original modeling, which should be used before saving the model in order to use the canonical transformers modeling:"),xo.forEach(o),Xt=u(e),v(ne.$$.fragment,e),Vt=u(e),H=a(e,"P",{});var Ao=l(H);nr=i(Ao,"As of PyTorch 2.0, the attention fastpath is supported for both encoders and decoders. The list of supported architectures can be found "),fe=a(Ao,"A",{href:!0,rel:!0});var Ka=l(fe);fr=i(Ka,"here"),Ka.forEach(o),mr=i(Ao,"."),Ao.forEach(o),Kt=u(e),I=a(e,"H2",{class:!0});var ko=l(I);z=a(ko,"A",{id:!0,class:!0,href:!0});var Za=l(z);nt=a(Za,"SPAN",{});var Ja=l(nt);v(me.$$.fragment,Ja),Ja.forEach(o),Za.forEach(o),pr=u(ko),Me=a(ko,"SPAN",{});var ka=l(Me);ft=a(ka,"CODE",{});var Qa=l(ft);ur=i(Qa,"bitsandbytes"),Qa.forEach(o),dr=i(ka," integration for Int8 mixed-precision matrix decomposition"),ka.forEach(o),ko.forEach(o),Zt=u(e),v(R.$$.fragment,e),Jt=u(e),h=a(e,"P",{});var P=l(h);hr=i(P,"From the paper "),pe=a(P,"A",{href:!0,rel:!0});var Ya=l(pe);mt=a(Ya,"CODE",{});var el=l(mt);cr=i(el,"LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),el.forEach(o),Ya.forEach(o),br=i(P,`, we support Hugging Face integration for all models in the Hub with a few lines of code.
The method reduces `),pt=a(P,"CODE",{});var tl=l(pt);_r=i(tl,"nn.Linear"),tl.forEach(o),vr=i(P," size by 2 for "),ut=a(P,"CODE",{});var ol=l(ut);gr=i(ol,"float16"),ol.forEach(o),wr=i(P," and "),dt=a(P,"CODE",{});var rl=l(dt);yr=i(rl,"bfloat16"),rl.forEach(o),$r=i(P," weights and by 4 for "),ht=a(P,"CODE",{});var al=l(ht);Er=i(al,"float32"),al.forEach(o),Pr=i(P," weights, with close to no impact to the quality by operating on the outliers in half-precision."),P.forEach(o),Qt=u(e),Ce=a(e,"P",{});var ll=l(Ce);qe=a(ll,"IMG",{src:!0,alt:!0}),ll.forEach(o),Yt=u(e),M=a(e,"P",{});var Je=l(M);xr=i(Je,`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),ue=a(Je,"A",{href:!0,rel:!0});var sl=l(ue);Ar=i(sl,"paper"),sl.forEach(o),kr=i(Je," or our "),de=a(Je,"A",{href:!0,rel:!0});var il=l(de);Tr=i(il,"blogpost about the integration"),il.forEach(o),Gr=i(Je,"."),Je.forEach(o),eo=u(e),Ue=a(e,"P",{});var nl=l(Ue);Ie=a(nl,"IMG",{src:!0,alt:!0}),nl.forEach(o),to=u(e),W=a(e,"P",{});var To=l(W);Mr=i(To,`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),Oe=a(To,"A",{href:!0});var fl=l(Oe);Cr=i(fl,"Google colab"),fl.forEach(o),qr=i(To,"."),To.forEach(o),oo=u(e),O=a(e,"H3",{class:!0});var Go=l(O);X=a(Go,"A",{id:!0,class:!0,href:!0});var ml=l(X);ct=a(ml,"SPAN",{});var pl=l(ct);v(he.$$.fragment,pl),pl.forEach(o),ml.forEach(o),Ur=u(Go),bt=a(Go,"SPAN",{});var ul=l(bt);Ir=i(ul,"Requirements"),ul.forEach(o),Go.forEach(o),ro=u(e),C=a(e,"UL",{});var Qe=l(C);L=a(Qe,"LI",{});var Ye=l(L);Or=i(Ye,"If you have "),_t=a(Ye,"CODE",{});var dl=l(_t);Lr=i(dl,"bitsandbytes<0.37.0"),dl.forEach(o),Br=i(Ye,", make sure you run on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100). For "),vt=a(Ye,"CODE",{});var hl=l(vt);jr=i(hl,"bitsandbytes>=0.37.0"),hl.forEach(o),Dr=i(Ye,", all GPUs should be supported."),Ye.forEach(o),Sr=u(Qe),V=a(Qe,"LI",{});var Bt=l(V);Fr=i(Bt,"Install the correct version of "),gt=a(Bt,"CODE",{});var cl=l(gt);Nr=i(cl,"bitsandbytes"),cl.forEach(o),Hr=i(Bt,` by running:
`),wt=a(Bt,"CODE",{});var bl=l(wt);zr=i(bl,"pip install bitsandbytes>=0.31.5"),bl.forEach(o),Bt.forEach(o),Rr=u(Qe),K=a(Qe,"LI",{});var jt=l(K);Wr=i(jt,"Install "),yt=a(jt,"CODE",{});var _l=l(yt);Xr=i(_l,"accelerate"),_l.forEach(o),Vr=u(jt),$t=a(jt,"CODE",{});var vl=l($t);Kr=i(vl,"pip install accelerate>=0.12.0"),vl.forEach(o),jt.forEach(o),Qe.forEach(o),ao=u(e),B=a(e,"H3",{class:!0});var Mo=l(B);Z=a(Mo,"A",{id:!0,class:!0,href:!0});var gl=l(Z);Et=a(gl,"SPAN",{});var wl=l(Et);v(ce.$$.fragment,wl),wl.forEach(o),gl.forEach(o),Zr=u(Mo),Pt=a(Mo,"SPAN",{});var yl=l(Pt);Jr=i(yl,"Running mixed-Int8 models - single GPU setup"),yl.forEach(o),Mo.forEach(o),lo=u(e),Le=a(e,"P",{});var $l=l(Le);Qr=i($l,"After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),$l.forEach(o),so=u(e),v(be.$$.fragment,e),io=u(e),Be=a(e,"P",{});var El=l(Be);Yr=i(El,"For text generation, we recommend:"),El.forEach(o),no=u(e),J=a(e,"UL",{});var Co=l(J);b=a(Co,"LI",{});var x=l(b);ea=i(x,"using the model\u2019s "),xt=a(x,"CODE",{});var Pl=l(xt);ta=i(Pl,"generate()"),Pl.forEach(o),oa=i(x," method instead of the "),At=a(x,"CODE",{});var xl=l(At);ra=i(xl,"pipeline()"),xl.forEach(o),aa=i(x," function. Although inference is possible with the "),kt=a(x,"CODE",{});var Al=l(kt);la=i(Al,"pipeline()"),Al.forEach(o),sa=i(x," function, it is not optimized for mixed-8bit models, and will be slower than using the "),Tt=a(x,"CODE",{});var kl=l(Tt);ia=i(kl,"generate()"),kl.forEach(o),na=i(x," method. Moreover, some sampling strategies are like nucleaus sampling are not supported by the "),Gt=a(x,"CODE",{});var Tl=l(Gt);fa=i(Tl,"pipeline()"),Tl.forEach(o),ma=i(x," function for mixed-8bit models."),x.forEach(o),pa=u(Co),Mt=a(Co,"LI",{});var Gl=l(Mt);ua=i(Gl,"placing all inputs on the same device as the model."),Gl.forEach(o),Co.forEach(o),fo=u(e),je=a(e,"P",{});var Ml=l(je);da=i(Ml,"Here is a simple example:"),Ml.forEach(o),mo=u(e),v(_e.$$.fragment,e),po=u(e),j=a(e,"H3",{class:!0});var qo=l(j);Q=a(qo,"A",{id:!0,class:!0,href:!0});var Cl=l(Q);Ct=a(Cl,"SPAN",{});var ql=l(Ct);v(ve.$$.fragment,ql),ql.forEach(o),Cl.forEach(o),ha=u(qo),qt=a(qo,"SPAN",{});var Ul=l(qt);ca=i(Ul,"Running mixed-int8 models - multi GPU setup"),Ul.forEach(o),qo.forEach(o),uo=u(e),De=a(e,"P",{});var Il=l(De);ba=i(Il,"The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Il.forEach(o),ho=u(e),v(ge.$$.fragment,e),co=u(e),q=a(e,"P",{});var et=l(q);_a=i(et,"But you can control the GPU RAM you want to allocate on each GPU using "),Ut=a(et,"CODE",{});var Ol=l(Ut);va=i(Ol,"accelerate"),Ol.forEach(o),ga=i(et,". Use the "),It=a(et,"CODE",{});var Ll=l(It);wa=i(Ll,"max_memory"),Ll.forEach(o),ya=i(et," argument as follows:"),et.forEach(o),bo=u(e),v(we.$$.fragment,e),_o=u(e),Se=a(e,"P",{});var Bl=l(Se);$a=i(Bl,"In this example, the first GPU will use 1GB of memory and the second 2GB."),Bl.forEach(o),vo=u(e),D=a(e,"H3",{class:!0});var Uo=l(D);Y=a(Uo,"A",{id:!0,class:!0,href:!0});var jl=l(Y);Ot=a(jl,"SPAN",{});var Dl=l(Ot);v(ye.$$.fragment,Dl),Dl.forEach(o),jl.forEach(o),Ea=u(Uo),Lt=a(Uo,"SPAN",{});var Sl=l(Lt);Pa=i(Sl,"Colab demos"),Sl.forEach(o),Uo.forEach(o),go=u(e),Fe=a(e,"P",{});var Fl=l(Fe);xa=i(Fl,`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),Fl.forEach(o),wo=u(e),Ne=a(e,"P",{});var Nl=l(Ne);$e=a(Nl,"A",{href:!0,rel:!0});var Hl=l($e);He=a(Hl,"IMG",{src:!0,alt:!0}),Hl.forEach(o),Nl.forEach(o),yo=u(e),ze=a(e,"P",{});var zl=l(ze);Aa=i(zl,"Or this demo for BLOOM-3B:"),zl.forEach(o),$o=u(e),Re=a(e,"P",{});var Rl=l(Re);Ee=a(Rl,"A",{href:!0,rel:!0});var Wl=l(Ee);We=a(Wl,"IMG",{src:!0,alt:!0}),Wl.forEach(o),Rl.forEach(o),this.h()},h(){m(d,"name","hf:doc:metadata"),m(d,"content",JSON.stringify(ts)),m(E,"id","efficient-inference-on-a-single-gpu"),m(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(E,"href","#efficient-inference-on-a-single-gpu"),m(c,"class","relative group"),m(Ae,"href","perf_train_gpu_one"),m(ke,"href","perf_infer_cpu"),m(F,"id","better-transformer-pytorchnative-transformer-fastpath"),m(F,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(F,"href","#better-transformer-pytorchnative-transformer-fastpath"),m(U,"class","relative group"),m(oe,"href","https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/"),m(oe,"rel","nofollow"),m(re,"href","https://huggingface.co/docs/optimum/bettertransformer/overview"),m(re,"rel","nofollow"),m(ae,"href","https://pytorch.org/docs/stable/nested.html"),m(ae,"rel","nofollow"),m(le,"href","https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2"),m(le,"rel","nofollow"),m(se,"href","https://github.com/huggingface/optimum"),m(se,"rel","nofollow"),m(Te,"href","/docs/transformers/v4.29.0/en/main_classes/model#transformers.PreTrainedModel.to_bettertransformer"),m(Ge,"href","/docs/transformers/v4.29.0/en/main_classes/model#transformers.PreTrainedModel.reverse_bettertransformer"),m(fe,"href","https://huggingface.co/docs/optimum/bettertransformer/overview#supported-models"),m(fe,"rel","nofollow"),m(z,"id","bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),m(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(z,"href","#bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),m(I,"class","relative group"),m(pe,"href","https://arxiv.org/abs/2208.07339"),m(pe,"rel","nofollow"),Io(qe.src,Ta="https://s3.amazonaws.com/moonup/production/uploads/1659861207959-62441d1d9fdefb55a0b7d12c.png")||m(qe,"src",Ta),m(qe,"alt","HFxbitsandbytes.png"),m(ue,"href","https://arxiv.org/abs/2208.07339"),m(ue,"rel","nofollow"),m(de,"href","https://huggingface.co/blog/hf-bitsandbytes-integration"),m(de,"rel","nofollow"),Io(Ie.src,Ga="https://s3.amazonaws.com/moonup/production/uploads/1660567469965-62441d1d9fdefb55a0b7d12c.gif")||m(Ie,"src",Ga),m(Ie,"alt","MixedInt8.gif"),m(Oe,"href","#colab-demos"),m(X,"id","requirements"),m(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(X,"href","#requirements"),m(O,"class","relative group"),m(Z,"id","running-mixedint8-models-single-gpu-setup"),m(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Z,"href","#running-mixedint8-models-single-gpu-setup"),m(B,"class","relative group"),m(Q,"id","running-mixedint8-models-multi-gpu-setup"),m(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Q,"href","#running-mixedint8-models-multi-gpu-setup"),m(j,"class","relative group"),m(Y,"id","colab-demos"),m(Y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Y,"href","#colab-demos"),m(D,"class","relative group"),Io(He.src,Ma="https://colab.research.google.com/assets/colab-badge.svg")||m(He,"src",Ma),m(He,"alt","Open In Colab: T5-11b demo"),m($e,"href","https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing"),m($e,"rel","nofollow"),Io(We.src,Ca="https://colab.research.google.com/assets/colab-badge.svg")||m(We,"src",Ca),m(We,"alt","Open In Colab: BLOOM-3b demo"),m(Ee,"href","https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing"),m(Ee,"rel","nofollow")},m(e,n){t(document.head,d),f(e,S,n),f(e,c,n),t(c,E),t(E,ot),g(ee,ot,null),t(c,Oo),t(c,rt),t(rt,Lo),f(e,St,n),f(e,A,n),t(A,Bo),t(A,Ae),t(Ae,jo),t(A,Do),t(A,ke),t(ke,So),t(A,Fo),f(e,Ft,n),f(e,U,n),t(U,F),t(F,at),g(te,at,null),t(U,No),t(U,lt),t(lt,Ho),f(e,Nt,n),f(e,k,n),t(k,zo),t(k,oe),t(oe,st),t(st,Ro),t(k,Wo),t(k,re),t(re,Xo),t(k,Vo),f(e,Ht,n),f(e,T,n),t(T,Ko),t(T,ae),t(ae,Zo),t(T,Jo),t(T,le),t(le,Qo),t(T,Yo),f(e,zt,n),f(e,G,n),t(G,er),t(G,se),t(se,it),t(it,tr),t(G,or),t(G,Te),t(Te,rr),t(G,ar),f(e,Rt,n),g(ie,e,n),f(e,Wt,n),f(e,N,n),t(N,lr),t(N,Ge),t(Ge,sr),t(N,ir),f(e,Xt,n),g(ne,e,n),f(e,Vt,n),f(e,H,n),t(H,nr),t(H,fe),t(fe,fr),t(H,mr),f(e,Kt,n),f(e,I,n),t(I,z),t(z,nt),g(me,nt,null),t(I,pr),t(I,Me),t(Me,ft),t(ft,ur),t(Me,dr),f(e,Zt,n),g(R,e,n),f(e,Jt,n),f(e,h,n),t(h,hr),t(h,pe),t(pe,mt),t(mt,cr),t(h,br),t(h,pt),t(pt,_r),t(h,vr),t(h,ut),t(ut,gr),t(h,wr),t(h,dt),t(dt,yr),t(h,$r),t(h,ht),t(ht,Er),t(h,Pr),f(e,Qt,n),f(e,Ce,n),t(Ce,qe),f(e,Yt,n),f(e,M,n),t(M,xr),t(M,ue),t(ue,Ar),t(M,kr),t(M,de),t(de,Tr),t(M,Gr),f(e,eo,n),f(e,Ue,n),t(Ue,Ie),f(e,to,n),f(e,W,n),t(W,Mr),t(W,Oe),t(Oe,Cr),t(W,qr),f(e,oo,n),f(e,O,n),t(O,X),t(X,ct),g(he,ct,null),t(O,Ur),t(O,bt),t(bt,Ir),f(e,ro,n),f(e,C,n),t(C,L),t(L,Or),t(L,_t),t(_t,Lr),t(L,Br),t(L,vt),t(vt,jr),t(L,Dr),t(C,Sr),t(C,V),t(V,Fr),t(V,gt),t(gt,Nr),t(V,Hr),t(V,wt),t(wt,zr),t(C,Rr),t(C,K),t(K,Wr),t(K,yt),t(yt,Xr),t(K,Vr),t(K,$t),t($t,Kr),f(e,ao,n),f(e,B,n),t(B,Z),t(Z,Et),g(ce,Et,null),t(B,Zr),t(B,Pt),t(Pt,Jr),f(e,lo,n),f(e,Le,n),t(Le,Qr),f(e,so,n),g(be,e,n),f(e,io,n),f(e,Be,n),t(Be,Yr),f(e,no,n),f(e,J,n),t(J,b),t(b,ea),t(b,xt),t(xt,ta),t(b,oa),t(b,At),t(At,ra),t(b,aa),t(b,kt),t(kt,la),t(b,sa),t(b,Tt),t(Tt,ia),t(b,na),t(b,Gt),t(Gt,fa),t(b,ma),t(J,pa),t(J,Mt),t(Mt,ua),f(e,fo,n),f(e,je,n),t(je,da),f(e,mo,n),g(_e,e,n),f(e,po,n),f(e,j,n),t(j,Q),t(Q,Ct),g(ve,Ct,null),t(j,ha),t(j,qt),t(qt,ca),f(e,uo,n),f(e,De,n),t(De,ba),f(e,ho,n),g(ge,e,n),f(e,co,n),f(e,q,n),t(q,_a),t(q,Ut),t(Ut,va),t(q,ga),t(q,It),t(It,wa),t(q,ya),f(e,bo,n),g(we,e,n),f(e,_o,n),f(e,Se,n),t(Se,$a),f(e,vo,n),f(e,D,n),t(D,Y),t(Y,Ot),g(ye,Ot,null),t(D,Ea),t(D,Lt),t(Lt,Pa),f(e,go,n),f(e,Fe,n),t(Fe,xa),f(e,wo,n),f(e,Ne,n),t(Ne,$e),t($e,He),f(e,yo,n),f(e,ze,n),t(ze,Aa),f(e,$o,n),f(e,Re,n),t(Re,Ee),t(Ee,We),Eo=!0},p(e,[n]){const Pe={};n&2&&(Pe.$$scope={dirty:n,ctx:e}),R.$set(Pe)},i(e){Eo||(w(ee.$$.fragment,e),w(te.$$.fragment,e),w(ie.$$.fragment,e),w(ne.$$.fragment,e),w(me.$$.fragment,e),w(R.$$.fragment,e),w(he.$$.fragment,e),w(ce.$$.fragment,e),w(be.$$.fragment,e),w(_e.$$.fragment,e),w(ve.$$.fragment,e),w(ge.$$.fragment,e),w(we.$$.fragment,e),w(ye.$$.fragment,e),Eo=!0)},o(e){y(ee.$$.fragment,e),y(te.$$.fragment,e),y(ie.$$.fragment,e),y(ne.$$.fragment,e),y(me.$$.fragment,e),y(R.$$.fragment,e),y(he.$$.fragment,e),y(ce.$$.fragment,e),y(be.$$.fragment,e),y(_e.$$.fragment,e),y(ve.$$.fragment,e),y(ge.$$.fragment,e),y(we.$$.fragment,e),y(ye.$$.fragment,e),Eo=!1},d(e){o(d),e&&o(S),e&&o(c),$(ee),e&&o(St),e&&o(A),e&&o(Ft),e&&o(U),$(te),e&&o(Nt),e&&o(k),e&&o(Ht),e&&o(T),e&&o(zt),e&&o(G),e&&o(Rt),$(ie,e),e&&o(Wt),e&&o(N),e&&o(Xt),$(ne,e),e&&o(Vt),e&&o(H),e&&o(Kt),e&&o(I),$(me),e&&o(Zt),$(R,e),e&&o(Jt),e&&o(h),e&&o(Qt),e&&o(Ce),e&&o(Yt),e&&o(M),e&&o(eo),e&&o(Ue),e&&o(to),e&&o(W),e&&o(oo),e&&o(O),$(he),e&&o(ro),e&&o(C),e&&o(ao),e&&o(B),$(ce),e&&o(lo),e&&o(Le),e&&o(so),$(be,e),e&&o(io),e&&o(Be),e&&o(no),e&&o(J),e&&o(fo),e&&o(je),e&&o(mo),$(_e,e),e&&o(po),e&&o(j),$(ve),e&&o(uo),e&&o(De),e&&o(ho),$(ge,e),e&&o(co),e&&o(q),e&&o(bo),$(we,e),e&&o(_o),e&&o(Se),e&&o(vo),e&&o(D),$(ye),e&&o(go),e&&o(Fe),e&&o(wo),e&&o(Ne),e&&o(yo),e&&o(ze),e&&o($o),e&&o(Re)}}}const ts={local:"efficient-inference-on-a-single-gpu",sections:[{local:"better-transformer-pytorchnative-transformer-fastpath",title:"Better Transformer: PyTorch-native transformer fastpath"},{local:"bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition",sections:[{local:"requirements",title:"Requirements"},{local:"running-mixedint8-models-single-gpu-setup",title:"Running mixed-Int8 models - single GPU setup"},{local:"running-mixedint8-models-multi-gpu-setup",title:"Running mixed-int8 models - multi GPU setup"},{local:"colab-demos",title:"Colab demos"}],title:"`bitsandbytes` integration for Int8 mixed-precision matrix decomposition"}],title:"Efficient Inference on a Single GPU"};function os(Dt){return Jl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class is extends Xl{constructor(d){super();Vl(this,d,os,es,Kl,{})}}export{is as default,ts as metadata};
