import{S as Hn,i as Ln,s as Kn,e as l,k as m,w as h,t as a,M as ef,c as r,d as o,m as u,a as s,x as d,h as i,b as p,N as cr,G as t,g as f,y as b,q as v,o as y,B as g,v as tf}from"../chunks/vendor-hf-doc-builder.js";import{T as Dn}from"../chunks/Tip-hf-doc-builder.js";import{I as P}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as D}from"../chunks/CodeBlock-hf-doc-builder.js";function of(et){let c,M;return{c(){c=l("p"),M=a("Note that this feature can also be used in a multi GPU setup.")},l(_){c=r(_,"P",{});var w=s(c);M=i(w,"Note that this feature can also be used in a multi GPU setup."),w.forEach(o)},m(_,w){f(_,c,w),t(c,M)},d(_){_&&o(c)}}}function lf(et){let c,M;return{c(){c=l("p"),M=a("Note that this feature can also be used in a multi GPU setup.")},l(_){c=r(_,"P",{});var w=s(c);M=i(w,"Note that this feature can also be used in a multi GPU setup."),w.forEach(o)},m(_,w){f(_,c,w),t(c,M)},d(_){_&&o(c)}}}function rf(et){let c,M,_,w,Rt,we,hr,jt,dr,Do,G,br,tt,vr,yr,ot,gr,_r,Ho,R,H,Nt,Ee,wr,qt,Er,Lo,k,$r,$e,Vt,Mr,Ur,Me,Tr,Pr,Ko,J,Gr,Ue,kr,Jr,Te,xr,Cr,el,x,Ir,Pe,Bt,Wr,Fr,lt,Ar,Zr,tl,Ge,ol,L,Rr,rt,jr,Nr,ll,ke,rl,K,qr,Je,Vr,Br,sl,j,ee,Xt,xe,Xr,st,Yt,Yr,Sr,al,te,Or,St,zr,Qr,il,oe,nl,N,le,Ot,Ce,Dr,zt,Hr,fl,C,Qt,re,Lr,Dt,Kr,es,Ht,ts,os,Lt,se,ls,Kt,rs,ss,eo,as,is,to,ae,ns,oo,fs,ps,lo,ms,pl,q,ie,ro,Ie,us,so,cs,ml,at,hs,ul,We,cl,I,ds,ao,bs,vs,io,ys,gs,hl,V,ne,no,Fe,_s,fo,ws,dl,it,Es,bl,Ae,vl,W,$s,po,Ms,Us,mo,Ts,Ps,yl,Ze,gl,nt,Gs,_l,B,fe,uo,Re,ks,co,Js,wl,pe,xs,ft,Cs,Is,El,X,me,ho,je,Ws,pt,bo,Fs,As,$l,ue,Ml,E,Zs,Ne,vo,Rs,js,yo,Ns,qs,go,Vs,Bs,_o,Xs,Ys,wo,Ss,Os,Ul,mt,ut,La,Tl,F,zs,qe,Qs,Ds,Ve,Hs,Ls,Pl,ct,ht,Ka,Gl,ce,Ks,dt,ea,ta,kl,Y,he,Eo,Be,oa,$o,la,Jl,A,S,ra,Mo,sa,aa,Uo,ia,na,fa,de,pa,To,ma,ua,Po,ca,ha,be,da,Go,ba,va,ko,ya,xl,O,ve,Jo,Xe,ga,xo,_a,Cl,bt,wa,Il,Ye,Wl,vt,Ea,Fl,ye,$,$a,Co,Ma,Ua,Io,Ta,Pa,Wo,Ga,ka,Fo,Ja,xa,Ao,Ca,Ia,Wa,Zo,Fa,Al,yt,Aa,Zl,Se,Rl,z,ge,Ro,Oe,Za,jo,Ra,jl,gt,ja,Nl,ze,ql,Z,Na,No,qa,Va,qo,Ba,Xa,Vl,Qe,Bl,_t,Ya,Xl,Q,_e,Vo,De,Sa,Bo,Oa,Yl,wt,za,Sl,Et,He,$t,ei,Ol,Mt,Qa,zl,Ut,Le,Tt,ti,Ql;return we=new P({}),Ee=new P({}),Ge=new D({props:{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC50b19iZXR0ZXJ0cmFuc2Zvcm1lcigp",highlighted:"model = model.to_bettertransformer()"}}),ke=new D({props:{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC5yZXZlcnNlX2JldHRlcnRyYW5zZm9ybWVyKCklMEFtb2RlbC5zYXZlX3ByZXRyYWluZWQoJTIyc2F2ZWRfbW9kZWwlMjIp",highlighted:`model = model.reverse_bettertransformer()
model.save_pretrained(<span class="hljs-string">&quot;saved_model&quot;</span>)`}}),xe=new P({}),oe=new Dn({props:{$$slots:{default:[of]},$$scope:{ctx:et}}}),Ce=new P({}),Ie=new P({}),We=new D({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`}}),Fe=new P({}),Ae=new D({props:{code:"bW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUp",highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`}}),Ze=new D({props:{code:"bWF4X21lbW9yeV9tYXBwaW5nJTIwJTNEJTIwJTdCMCUzQSUyMCUyMjYwME1CJTIyJTJDJTIwMSUzQSUyMCUyMjFHQiUyMiU3RCUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tM2IlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMjBtYXhfbWVtb3J5JTNEbWF4X21lbW9yeV9tYXBwaW5nJTBBKQ==",highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;600MB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;1GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`}}),Re=new P({}),je=new P({}),ue=new Dn({props:{$$slots:{default:[lf]},$$scope:{ctx:et}}}),Be=new P({}),Xe=new P({}),Ye=new D({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),Se=new D({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMmI1JTIyJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSklMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIySGVsbG8lMkMlMjBteSUyMGxsYW1hJTIwaXMlMjBjdXRlJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKSUwQW91dHB1dHMlMjAlM0QlMjB0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKGdlbmVyYXRlZF9pZHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_name)
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)

prompt = <span class="hljs-string">&quot;Hello, my llama is cute&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
generated_ids = model.generate(**inputs)
outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)`}}),Oe=new P({}),ze=new D({props:{code:"bW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUp",highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),Qe=new D({props:{code:"bWF4X21lbW9yeV9tYXBwaW5nJTIwJTNEJTIwJTdCMCUzQSUyMCUyMjFHQiUyMiUyQyUyMDElM0ElMjAlMjIyR0IlMjIlN0QlMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTNiJTIyJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBtb2RlbF9uYW1lJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlJTJDJTIwbWF4X21lbW9yeSUzRG1heF9tZW1vcnlfbWFwcGluZyUwQSk=",highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;1GB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;2GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`}}),De=new P({}),{c(){c=l("meta"),M=m(),_=l("h1"),w=l("a"),Rt=l("span"),h(we.$$.fragment),hr=m(),jt=l("span"),dr=a("Efficient Inference on a Single GPU"),Do=m(),G=l("p"),br=a("In addition to this guide, relevant information can be found as well in "),tt=l("a"),vr=a("the guide for training on a single GPU"),yr=a(" and "),ot=l("a"),gr=a("the guide for inference on CPUs"),_r=a("."),Ho=m(),R=l("h2"),H=l("a"),Nt=l("span"),h(Ee.$$.fragment),wr=m(),qt=l("span"),Er=a("Better Transformer: PyTorch-native transformer fastpath"),Lo=m(),k=l("p"),$r=a("PyTorch-native "),$e=l("a"),Vt=l("code"),Mr=a("nn.MultiHeadAttention"),Ur=a(" attention fastpath, called BetterTransformer, can be used with Transformers through the integration in the "),Me=l("a"),Tr=a("\u{1F917} Optimum library"),Pr=a("."),Ko=m(),J=l("p"),Gr=a("PyTorch\u2019s attention fastpath allows to speed up inference through kernel fusions and the use of "),Ue=l("a"),kr=a("nested tensors"),Jr=a(". Detailed benchmarks can be found in "),Te=l("a"),xr=a("this blog post"),Cr=a("."),el=m(),x=l("p"),Ir=a("After installing the "),Pe=l("a"),Bt=l("code"),Wr=a("optimum"),Fr=a(" package, to use Better Transformer during inference, the relevant internal modules are replaced by calling "),lt=l("a"),Ar=a("to_bettertransformer()"),Zr=a(":"),tl=m(),h(Ge.$$.fragment),ol=m(),L=l("p"),Rr=a("The method "),rt=l("a"),jr=a("reverse_bettertransformer()"),Nr=a(" allows to go back to the original modeling, which should be used before saving the model in order to use the canonical transformers modeling:"),ll=m(),h(ke.$$.fragment),rl=m(),K=l("p"),qr=a("As of PyTorch 2.0, the attention fastpath is supported for both encoders and decoders. The list of supported architectures can be found "),Je=l("a"),Vr=a("here"),Br=a("."),sl=m(),j=l("h2"),ee=l("a"),Xt=l("span"),h(xe.$$.fragment),Xr=m(),st=l("span"),Yt=l("code"),Yr=a("bitsandbytes"),Sr=a(" integration for FP4 mixed-precision inference"),al=m(),te=l("p"),Or=a("You can install "),St=l("code"),zr=a("bitsandbytes"),Qr=a(" and benefit from easy model compression on GPUs. Using FP4 quantization you can expect to reduce up to 8x the model size compared to its native full precision version. Check out below how to get started."),il=m(),h(oe.$$.fragment),nl=m(),N=l("h3"),le=l("a"),Ot=l("span"),h(Ce.$$.fragment),Dr=m(),zt=l("span"),Hr=a("Requirements"),fl=m(),C=l("ul"),Qt=l("li"),re=l("p"),Lr=a("Latest "),Dt=l("code"),Kr=a("bitsandbytes"),es=a(` library
`),Ht=l("code"),ts=a("pip install bitsandbytes>=0.39.0"),os=m(),Lt=l("li"),se=l("p"),ls=a("Install latest "),Kt=l("code"),rs=a("accelerate"),ss=a(` from source
`),eo=l("code"),as=a("pip install git+https://github.com/huggingface/accelerate.git"),is=m(),to=l("li"),ae=l("p"),ns=a("Install latest "),oo=l("code"),fs=a("transformers"),ps=a(` from source
`),lo=l("code"),ms=a("pip install git+https://github.com/huggingface/transformers.git"),pl=m(),q=l("h3"),ie=l("a"),ro=l("span"),h(Ie.$$.fragment),us=m(),so=l("span"),cs=a("Running FP4 models - single GPU setup - Quickstart"),ml=m(),at=l("p"),hs=a("You can quickly run a FP4 model on a single GPU by running the following code:"),ul=m(),h(We.$$.fragment),cl=m(),I=l("p"),ds=a("Note that "),ao=l("code"),bs=a("device_map"),vs=a(" is optional but setting "),io=l("code"),ys=a("device_map = 'auto'"),gs=a(" is prefered for inference as it will dispatch efficiently the model on the available ressources."),hl=m(),V=l("h3"),ne=l("a"),no=l("span"),h(Fe.$$.fragment),_s=m(),fo=l("span"),ws=a("Running FP4 models - multi GPU setup"),dl=m(),it=l("p"),Es=a("The way to load your mixed 4-bit model in multiple GPUs is as follows (same command as single GPU setup):"),bl=m(),h(Ae.$$.fragment),vl=m(),W=l("p"),$s=a("But you can control the GPU RAM you want to allocate on each GPU using "),po=l("code"),Ms=a("accelerate"),Us=a(". Use the "),mo=l("code"),Ts=a("max_memory"),Ps=a(" argument as follows:"),yl=m(),h(Ze.$$.fragment),gl=m(),nt=l("p"),Gs=a("In this example, the first GPU will use 600MB of memory and the second 1GB."),_l=m(),B=l("h3"),fe=l("a"),uo=l("span"),h(Re.$$.fragment),ks=m(),co=l("span"),Js=a("Advanced usage"),wl=m(),pe=l("p"),xs=a("For more advanced usage of this method, please have a look at the "),ft=l("a"),Cs=a("quantization"),Is=a(" documentation page."),El=m(),X=l("h2"),me=l("a"),ho=l("span"),h(je.$$.fragment),Ws=m(),pt=l("span"),bo=l("code"),Fs=a("bitsandbytes"),As=a(" integration for Int8 mixed-precision matrix decomposition"),$l=m(),h(ue.$$.fragment),Ml=m(),E=l("p"),Zs=a("From the paper "),Ne=l("a"),vo=l("code"),Rs=a("LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),js=a(`, we support Hugging Face integration for all models in the Hub with a few lines of code.
The method reduces `),yo=l("code"),Ns=a("nn.Linear"),qs=a(" size by 2 for "),go=l("code"),Vs=a("float16"),Bs=a(" and "),_o=l("code"),Xs=a("bfloat16"),Ys=a(" weights and by 4 for "),wo=l("code"),Ss=a("float32"),Os=a(" weights, with close to no impact to the quality by operating on the outliers in half-precision."),Ul=m(),mt=l("p"),ut=l("img"),Tl=m(),F=l("p"),zs=a(`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),qe=l("a"),Qs=a("paper"),Ds=a(" or our "),Ve=l("a"),Hs=a("blogpost about the integration"),Ls=a("."),Pl=m(),ct=l("p"),ht=l("img"),Gl=m(),ce=l("p"),Ks=a(`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),dt=l("a"),ea=a("Google colab"),ta=a("."),kl=m(),Y=l("h3"),he=l("a"),Eo=l("span"),h(Be.$$.fragment),oa=m(),$o=l("span"),la=a("Requirements"),Jl=m(),A=l("ul"),S=l("li"),ra=a("If you have "),Mo=l("code"),sa=a("bitsandbytes<0.37.0"),aa=a(", make sure you run on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100). For "),Uo=l("code"),ia=a("bitsandbytes>=0.37.0"),na=a(", all GPUs should be supported."),fa=m(),de=l("li"),pa=a("Install the correct version of "),To=l("code"),ma=a("bitsandbytes"),ua=a(` by running:
`),Po=l("code"),ca=a("pip install bitsandbytes>=0.31.5"),ha=m(),be=l("li"),da=a("Install "),Go=l("code"),ba=a("accelerate"),va=m(),ko=l("code"),ya=a("pip install accelerate>=0.12.0"),xl=m(),O=l("h3"),ve=l("a"),Jo=l("span"),h(Xe.$$.fragment),ga=m(),xo=l("span"),_a=a("Running mixed-Int8 models - single GPU setup"),Cl=m(),bt=l("p"),wa=a("After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),Il=m(),h(Ye.$$.fragment),Wl=m(),vt=l("p"),Ea=a("For text generation, we recommend:"),Fl=m(),ye=l("ul"),$=l("li"),$a=a("using the model\u2019s "),Co=l("code"),Ma=a("generate()"),Ua=a(" method instead of the "),Io=l("code"),Ta=a("pipeline()"),Pa=a(" function. Although inference is possible with the "),Wo=l("code"),Ga=a("pipeline()"),ka=a(" function, it is not optimized for mixed-8bit models, and will be slower than using the "),Fo=l("code"),Ja=a("generate()"),xa=a(" method. Moreover, some sampling strategies are like nucleaus sampling are not supported by the "),Ao=l("code"),Ca=a("pipeline()"),Ia=a(" function for mixed-8bit models."),Wa=m(),Zo=l("li"),Fa=a("placing all inputs on the same device as the model."),Al=m(),yt=l("p"),Aa=a("Here is a simple example:"),Zl=m(),h(Se.$$.fragment),Rl=m(),z=l("h3"),ge=l("a"),Ro=l("span"),h(Oe.$$.fragment),Za=m(),jo=l("span"),Ra=a("Running mixed-int8 models - multi GPU setup"),jl=m(),gt=l("p"),ja=a("The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Nl=m(),h(ze.$$.fragment),ql=m(),Z=l("p"),Na=a("But you can control the GPU RAM you want to allocate on each GPU using "),No=l("code"),qa=a("accelerate"),Va=a(". Use the "),qo=l("code"),Ba=a("max_memory"),Xa=a(" argument as follows:"),Vl=m(),h(Qe.$$.fragment),Bl=m(),_t=l("p"),Ya=a("In this example, the first GPU will use 1GB of memory and the second 2GB."),Xl=m(),Q=l("h3"),_e=l("a"),Vo=l("span"),h(De.$$.fragment),Sa=m(),Bo=l("span"),Oa=a("Colab demos"),Yl=m(),wt=l("p"),za=a(`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),Sl=m(),Et=l("p"),He=l("a"),$t=l("img"),Ol=m(),Mt=l("p"),Qa=a("Or this demo for BLOOM-3B:"),zl=m(),Ut=l("p"),Le=l("a"),Tt=l("img"),this.h()},l(e){const n=ef('[data-svelte="svelte-1phssyn"]',document.head);c=r(n,"META",{name:!0,content:!0}),n.forEach(o),M=u(e),_=r(e,"H1",{class:!0});var Ke=s(_);w=r(Ke,"A",{id:!0,class:!0,href:!0});var Xo=s(w);Rt=r(Xo,"SPAN",{});var oi=s(Rt);d(we.$$.fragment,oi),oi.forEach(o),Xo.forEach(o),hr=u(Ke),jt=r(Ke,"SPAN",{});var li=s(jt);dr=i(li,"Efficient Inference on a Single GPU"),li.forEach(o),Ke.forEach(o),Do=u(e),G=r(e,"P",{});var Pt=s(G);br=i(Pt,"In addition to this guide, relevant information can be found as well in "),tt=r(Pt,"A",{href:!0});var ri=s(tt);vr=i(ri,"the guide for training on a single GPU"),ri.forEach(o),yr=i(Pt," and "),ot=r(Pt,"A",{href:!0});var si=s(ot);gr=i(si,"the guide for inference on CPUs"),si.forEach(o),_r=i(Pt,"."),Pt.forEach(o),Ho=u(e),R=r(e,"H2",{class:!0});var Dl=s(R);H=r(Dl,"A",{id:!0,class:!0,href:!0});var ai=s(H);Nt=r(ai,"SPAN",{});var ii=s(Nt);d(Ee.$$.fragment,ii),ii.forEach(o),ai.forEach(o),wr=u(Dl),qt=r(Dl,"SPAN",{});var ni=s(qt);Er=i(ni,"Better Transformer: PyTorch-native transformer fastpath"),ni.forEach(o),Dl.forEach(o),Lo=u(e),k=r(e,"P",{});var Gt=s(k);$r=i(Gt,"PyTorch-native "),$e=r(Gt,"A",{href:!0,rel:!0});var fi=s($e);Vt=r(fi,"CODE",{});var pi=s(Vt);Mr=i(pi,"nn.MultiHeadAttention"),pi.forEach(o),fi.forEach(o),Ur=i(Gt," attention fastpath, called BetterTransformer, can be used with Transformers through the integration in the "),Me=r(Gt,"A",{href:!0,rel:!0});var mi=s(Me);Tr=i(mi,"\u{1F917} Optimum library"),mi.forEach(o),Pr=i(Gt,"."),Gt.forEach(o),Ko=u(e),J=r(e,"P",{});var kt=s(J);Gr=i(kt,"PyTorch\u2019s attention fastpath allows to speed up inference through kernel fusions and the use of "),Ue=r(kt,"A",{href:!0,rel:!0});var ui=s(Ue);kr=i(ui,"nested tensors"),ui.forEach(o),Jr=i(kt,". Detailed benchmarks can be found in "),Te=r(kt,"A",{href:!0,rel:!0});var ci=s(Te);xr=i(ci,"this blog post"),ci.forEach(o),Cr=i(kt,"."),kt.forEach(o),el=u(e),x=r(e,"P",{});var Jt=s(x);Ir=i(Jt,"After installing the "),Pe=r(Jt,"A",{href:!0,rel:!0});var hi=s(Pe);Bt=r(hi,"CODE",{});var di=s(Bt);Wr=i(di,"optimum"),di.forEach(o),hi.forEach(o),Fr=i(Jt," package, to use Better Transformer during inference, the relevant internal modules are replaced by calling "),lt=r(Jt,"A",{href:!0});var bi=s(lt);Ar=i(bi,"to_bettertransformer()"),bi.forEach(o),Zr=i(Jt,":"),Jt.forEach(o),tl=u(e),d(Ge.$$.fragment,e),ol=u(e),L=r(e,"P",{});var Hl=s(L);Rr=i(Hl,"The method "),rt=r(Hl,"A",{href:!0});var vi=s(rt);jr=i(vi,"reverse_bettertransformer()"),vi.forEach(o),Nr=i(Hl," allows to go back to the original modeling, which should be used before saving the model in order to use the canonical transformers modeling:"),Hl.forEach(o),ll=u(e),d(ke.$$.fragment,e),rl=u(e),K=r(e,"P",{});var Ll=s(K);qr=i(Ll,"As of PyTorch 2.0, the attention fastpath is supported for both encoders and decoders. The list of supported architectures can be found "),Je=r(Ll,"A",{href:!0,rel:!0});var yi=s(Je);Vr=i(yi,"here"),yi.forEach(o),Br=i(Ll,"."),Ll.forEach(o),sl=u(e),j=r(e,"H2",{class:!0});var Kl=s(j);ee=r(Kl,"A",{id:!0,class:!0,href:!0});var gi=s(ee);Xt=r(gi,"SPAN",{});var _i=s(Xt);d(xe.$$.fragment,_i),_i.forEach(o),gi.forEach(o),Xr=u(Kl),st=r(Kl,"SPAN",{});var Da=s(st);Yt=r(Da,"CODE",{});var wi=s(Yt);Yr=i(wi,"bitsandbytes"),wi.forEach(o),Sr=i(Da," integration for FP4 mixed-precision inference"),Da.forEach(o),Kl.forEach(o),al=u(e),te=r(e,"P",{});var er=s(te);Or=i(er,"You can install "),St=r(er,"CODE",{});var Ei=s(St);zr=i(Ei,"bitsandbytes"),Ei.forEach(o),Qr=i(er," and benefit from easy model compression on GPUs. Using FP4 quantization you can expect to reduce up to 8x the model size compared to its native full precision version. Check out below how to get started."),er.forEach(o),il=u(e),d(oe.$$.fragment,e),nl=u(e),N=r(e,"H3",{class:!0});var tr=s(N);le=r(tr,"A",{id:!0,class:!0,href:!0});var $i=s(le);Ot=r($i,"SPAN",{});var Mi=s(Ot);d(Ce.$$.fragment,Mi),Mi.forEach(o),$i.forEach(o),Dr=u(tr),zt=r(tr,"SPAN",{});var Ui=s(zt);Hr=i(Ui,"Requirements"),Ui.forEach(o),tr.forEach(o),fl=u(e),C=r(e,"UL",{});var xt=s(C);Qt=r(xt,"LI",{});var Ti=s(Qt);re=r(Ti,"P",{});var Yo=s(re);Lr=i(Yo,"Latest "),Dt=r(Yo,"CODE",{});var Pi=s(Dt);Kr=i(Pi,"bitsandbytes"),Pi.forEach(o),es=i(Yo,` library
`),Ht=r(Yo,"CODE",{});var Gi=s(Ht);ts=i(Gi,"pip install bitsandbytes>=0.39.0"),Gi.forEach(o),Yo.forEach(o),Ti.forEach(o),os=u(xt),Lt=r(xt,"LI",{});var ki=s(Lt);se=r(ki,"P",{});var So=s(se);ls=i(So,"Install latest "),Kt=r(So,"CODE",{});var Ji=s(Kt);rs=i(Ji,"accelerate"),Ji.forEach(o),ss=i(So,` from source
`),eo=r(So,"CODE",{});var xi=s(eo);as=i(xi,"pip install git+https://github.com/huggingface/accelerate.git"),xi.forEach(o),So.forEach(o),ki.forEach(o),is=u(xt),to=r(xt,"LI",{});var Ci=s(to);ae=r(Ci,"P",{});var Oo=s(ae);ns=i(Oo,"Install latest "),oo=r(Oo,"CODE",{});var Ii=s(oo);fs=i(Ii,"transformers"),Ii.forEach(o),ps=i(Oo,` from source
`),lo=r(Oo,"CODE",{});var Wi=s(lo);ms=i(Wi,"pip install git+https://github.com/huggingface/transformers.git"),Wi.forEach(o),Oo.forEach(o),Ci.forEach(o),xt.forEach(o),pl=u(e),q=r(e,"H3",{class:!0});var or=s(q);ie=r(or,"A",{id:!0,class:!0,href:!0});var Fi=s(ie);ro=r(Fi,"SPAN",{});var Ai=s(ro);d(Ie.$$.fragment,Ai),Ai.forEach(o),Fi.forEach(o),us=u(or),so=r(or,"SPAN",{});var Zi=s(so);cs=i(Zi,"Running FP4 models - single GPU setup - Quickstart"),Zi.forEach(o),or.forEach(o),ml=u(e),at=r(e,"P",{});var Ri=s(at);hs=i(Ri,"You can quickly run a FP4 model on a single GPU by running the following code:"),Ri.forEach(o),ul=u(e),d(We.$$.fragment,e),cl=u(e),I=r(e,"P",{});var Ct=s(I);ds=i(Ct,"Note that "),ao=r(Ct,"CODE",{});var ji=s(ao);bs=i(ji,"device_map"),ji.forEach(o),vs=i(Ct," is optional but setting "),io=r(Ct,"CODE",{});var Ni=s(io);ys=i(Ni,"device_map = 'auto'"),Ni.forEach(o),gs=i(Ct," is prefered for inference as it will dispatch efficiently the model on the available ressources."),Ct.forEach(o),hl=u(e),V=r(e,"H3",{class:!0});var lr=s(V);ne=r(lr,"A",{id:!0,class:!0,href:!0});var qi=s(ne);no=r(qi,"SPAN",{});var Vi=s(no);d(Fe.$$.fragment,Vi),Vi.forEach(o),qi.forEach(o),_s=u(lr),fo=r(lr,"SPAN",{});var Bi=s(fo);ws=i(Bi,"Running FP4 models - multi GPU setup"),Bi.forEach(o),lr.forEach(o),dl=u(e),it=r(e,"P",{});var Xi=s(it);Es=i(Xi,"The way to load your mixed 4-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Xi.forEach(o),bl=u(e),d(Ae.$$.fragment,e),vl=u(e),W=r(e,"P",{});var It=s(W);$s=i(It,"But you can control the GPU RAM you want to allocate on each GPU using "),po=r(It,"CODE",{});var Yi=s(po);Ms=i(Yi,"accelerate"),Yi.forEach(o),Us=i(It,". Use the "),mo=r(It,"CODE",{});var Si=s(mo);Ts=i(Si,"max_memory"),Si.forEach(o),Ps=i(It," argument as follows:"),It.forEach(o),yl=u(e),d(Ze.$$.fragment,e),gl=u(e),nt=r(e,"P",{});var Oi=s(nt);Gs=i(Oi,"In this example, the first GPU will use 600MB of memory and the second 1GB."),Oi.forEach(o),_l=u(e),B=r(e,"H3",{class:!0});var rr=s(B);fe=r(rr,"A",{id:!0,class:!0,href:!0});var zi=s(fe);uo=r(zi,"SPAN",{});var Qi=s(uo);d(Re.$$.fragment,Qi),Qi.forEach(o),zi.forEach(o),ks=u(rr),co=r(rr,"SPAN",{});var Di=s(co);Js=i(Di,"Advanced usage"),Di.forEach(o),rr.forEach(o),wl=u(e),pe=r(e,"P",{});var sr=s(pe);xs=i(sr,"For more advanced usage of this method, please have a look at the "),ft=r(sr,"A",{href:!0});var Hi=s(ft);Cs=i(Hi,"quantization"),Hi.forEach(o),Is=i(sr," documentation page."),sr.forEach(o),El=u(e),X=r(e,"H2",{class:!0});var ar=s(X);me=r(ar,"A",{id:!0,class:!0,href:!0});var Li=s(me);ho=r(Li,"SPAN",{});var Ki=s(ho);d(je.$$.fragment,Ki),Ki.forEach(o),Li.forEach(o),Ws=u(ar),pt=r(ar,"SPAN",{});var Ha=s(pt);bo=r(Ha,"CODE",{});var en=s(bo);Fs=i(en,"bitsandbytes"),en.forEach(o),As=i(Ha," integration for Int8 mixed-precision matrix decomposition"),Ha.forEach(o),ar.forEach(o),$l=u(e),d(ue.$$.fragment,e),Ml=u(e),E=r(e,"P",{});var U=s(E);Zs=i(U,"From the paper "),Ne=r(U,"A",{href:!0,rel:!0});var tn=s(Ne);vo=r(tn,"CODE",{});var on=s(vo);Rs=i(on,"LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),on.forEach(o),tn.forEach(o),js=i(U,`, we support Hugging Face integration for all models in the Hub with a few lines of code.
The method reduces `),yo=r(U,"CODE",{});var ln=s(yo);Ns=i(ln,"nn.Linear"),ln.forEach(o),qs=i(U," size by 2 for "),go=r(U,"CODE",{});var rn=s(go);Vs=i(rn,"float16"),rn.forEach(o),Bs=i(U," and "),_o=r(U,"CODE",{});var sn=s(_o);Xs=i(sn,"bfloat16"),sn.forEach(o),Ys=i(U," weights and by 4 for "),wo=r(U,"CODE",{});var an=s(wo);Ss=i(an,"float32"),an.forEach(o),Os=i(U," weights, with close to no impact to the quality by operating on the outliers in half-precision."),U.forEach(o),Ul=u(e),mt=r(e,"P",{});var nn=s(mt);ut=r(nn,"IMG",{src:!0,alt:!0}),nn.forEach(o),Tl=u(e),F=r(e,"P",{});var Wt=s(F);zs=i(Wt,`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),qe=r(Wt,"A",{href:!0,rel:!0});var fn=s(qe);Qs=i(fn,"paper"),fn.forEach(o),Ds=i(Wt," or our "),Ve=r(Wt,"A",{href:!0,rel:!0});var pn=s(Ve);Hs=i(pn,"blogpost about the integration"),pn.forEach(o),Ls=i(Wt,"."),Wt.forEach(o),Pl=u(e),ct=r(e,"P",{});var mn=s(ct);ht=r(mn,"IMG",{src:!0,alt:!0}),mn.forEach(o),Gl=u(e),ce=r(e,"P",{});var ir=s(ce);Ks=i(ir,`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),dt=r(ir,"A",{href:!0});var un=s(dt);ea=i(un,"Google colab"),un.forEach(o),ta=i(ir,"."),ir.forEach(o),kl=u(e),Y=r(e,"H3",{class:!0});var nr=s(Y);he=r(nr,"A",{id:!0,class:!0,href:!0});var cn=s(he);Eo=r(cn,"SPAN",{});var hn=s(Eo);d(Be.$$.fragment,hn),hn.forEach(o),cn.forEach(o),oa=u(nr),$o=r(nr,"SPAN",{});var dn=s($o);la=i(dn,"Requirements"),dn.forEach(o),nr.forEach(o),Jl=u(e),A=r(e,"UL",{});var Ft=s(A);S=r(Ft,"LI",{});var At=s(S);ra=i(At,"If you have "),Mo=r(At,"CODE",{});var bn=s(Mo);sa=i(bn,"bitsandbytes<0.37.0"),bn.forEach(o),aa=i(At,", make sure you run on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100). For "),Uo=r(At,"CODE",{});var vn=s(Uo);ia=i(vn,"bitsandbytes>=0.37.0"),vn.forEach(o),na=i(At,", all GPUs should be supported."),At.forEach(o),fa=u(Ft),de=r(Ft,"LI",{});var zo=s(de);pa=i(zo,"Install the correct version of "),To=r(zo,"CODE",{});var yn=s(To);ma=i(yn,"bitsandbytes"),yn.forEach(o),ua=i(zo,` by running:
`),Po=r(zo,"CODE",{});var gn=s(Po);ca=i(gn,"pip install bitsandbytes>=0.31.5"),gn.forEach(o),zo.forEach(o),ha=u(Ft),be=r(Ft,"LI",{});var Qo=s(be);da=i(Qo,"Install "),Go=r(Qo,"CODE",{});var _n=s(Go);ba=i(_n,"accelerate"),_n.forEach(o),va=u(Qo),ko=r(Qo,"CODE",{});var wn=s(ko);ya=i(wn,"pip install accelerate>=0.12.0"),wn.forEach(o),Qo.forEach(o),Ft.forEach(o),xl=u(e),O=r(e,"H3",{class:!0});var fr=s(O);ve=r(fr,"A",{id:!0,class:!0,href:!0});var En=s(ve);Jo=r(En,"SPAN",{});var $n=s(Jo);d(Xe.$$.fragment,$n),$n.forEach(o),En.forEach(o),ga=u(fr),xo=r(fr,"SPAN",{});var Mn=s(xo);_a=i(Mn,"Running mixed-Int8 models - single GPU setup"),Mn.forEach(o),fr.forEach(o),Cl=u(e),bt=r(e,"P",{});var Un=s(bt);wa=i(Un,"After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),Un.forEach(o),Il=u(e),d(Ye.$$.fragment,e),Wl=u(e),vt=r(e,"P",{});var Tn=s(vt);Ea=i(Tn,"For text generation, we recommend:"),Tn.forEach(o),Fl=u(e),ye=r(e,"UL",{});var pr=s(ye);$=r(pr,"LI",{});var T=s($);$a=i(T,"using the model\u2019s "),Co=r(T,"CODE",{});var Pn=s(Co);Ma=i(Pn,"generate()"),Pn.forEach(o),Ua=i(T," method instead of the "),Io=r(T,"CODE",{});var Gn=s(Io);Ta=i(Gn,"pipeline()"),Gn.forEach(o),Pa=i(T," function. Although inference is possible with the "),Wo=r(T,"CODE",{});var kn=s(Wo);Ga=i(kn,"pipeline()"),kn.forEach(o),ka=i(T," function, it is not optimized for mixed-8bit models, and will be slower than using the "),Fo=r(T,"CODE",{});var Jn=s(Fo);Ja=i(Jn,"generate()"),Jn.forEach(o),xa=i(T," method. Moreover, some sampling strategies are like nucleaus sampling are not supported by the "),Ao=r(T,"CODE",{});var xn=s(Ao);Ca=i(xn,"pipeline()"),xn.forEach(o),Ia=i(T," function for mixed-8bit models."),T.forEach(o),Wa=u(pr),Zo=r(pr,"LI",{});var Cn=s(Zo);Fa=i(Cn,"placing all inputs on the same device as the model."),Cn.forEach(o),pr.forEach(o),Al=u(e),yt=r(e,"P",{});var In=s(yt);Aa=i(In,"Here is a simple example:"),In.forEach(o),Zl=u(e),d(Se.$$.fragment,e),Rl=u(e),z=r(e,"H3",{class:!0});var mr=s(z);ge=r(mr,"A",{id:!0,class:!0,href:!0});var Wn=s(ge);Ro=r(Wn,"SPAN",{});var Fn=s(Ro);d(Oe.$$.fragment,Fn),Fn.forEach(o),Wn.forEach(o),Za=u(mr),jo=r(mr,"SPAN",{});var An=s(jo);Ra=i(An,"Running mixed-int8 models - multi GPU setup"),An.forEach(o),mr.forEach(o),jl=u(e),gt=r(e,"P",{});var Zn=s(gt);ja=i(Zn,"The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Zn.forEach(o),Nl=u(e),d(ze.$$.fragment,e),ql=u(e),Z=r(e,"P",{});var Zt=s(Z);Na=i(Zt,"But you can control the GPU RAM you want to allocate on each GPU using "),No=r(Zt,"CODE",{});var Rn=s(No);qa=i(Rn,"accelerate"),Rn.forEach(o),Va=i(Zt,". Use the "),qo=r(Zt,"CODE",{});var jn=s(qo);Ba=i(jn,"max_memory"),jn.forEach(o),Xa=i(Zt," argument as follows:"),Zt.forEach(o),Vl=u(e),d(Qe.$$.fragment,e),Bl=u(e),_t=r(e,"P",{});var Nn=s(_t);Ya=i(Nn,"In this example, the first GPU will use 1GB of memory and the second 2GB."),Nn.forEach(o),Xl=u(e),Q=r(e,"H3",{class:!0});var ur=s(Q);_e=r(ur,"A",{id:!0,class:!0,href:!0});var qn=s(_e);Vo=r(qn,"SPAN",{});var Vn=s(Vo);d(De.$$.fragment,Vn),Vn.forEach(o),qn.forEach(o),Sa=u(ur),Bo=r(ur,"SPAN",{});var Bn=s(Bo);Oa=i(Bn,"Colab demos"),Bn.forEach(o),ur.forEach(o),Yl=u(e),wt=r(e,"P",{});var Xn=s(wt);za=i(Xn,`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),Xn.forEach(o),Sl=u(e),Et=r(e,"P",{});var Yn=s(Et);He=r(Yn,"A",{href:!0,rel:!0});var Sn=s(He);$t=r(Sn,"IMG",{src:!0,alt:!0}),Sn.forEach(o),Yn.forEach(o),Ol=u(e),Mt=r(e,"P",{});var On=s(Mt);Qa=i(On,"Or this demo for BLOOM-3B:"),On.forEach(o),zl=u(e),Ut=r(e,"P",{});var zn=s(Ut);Le=r(zn,"A",{href:!0,rel:!0});var Qn=s(Le);Tt=r(Qn,"IMG",{src:!0,alt:!0}),Qn.forEach(o),zn.forEach(o),this.h()},h(){p(c,"name","hf:doc:metadata"),p(c,"content",JSON.stringify(sf)),p(w,"id","efficient-inference-on-a-single-gpu"),p(w,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(w,"href","#efficient-inference-on-a-single-gpu"),p(_,"class","relative group"),p(tt,"href","perf_train_gpu_one"),p(ot,"href","perf_infer_cpu"),p(H,"id","better-transformer-pytorchnative-transformer-fastpath"),p(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(H,"href","#better-transformer-pytorchnative-transformer-fastpath"),p(R,"class","relative group"),p($e,"href","https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/"),p($e,"rel","nofollow"),p(Me,"href","https://huggingface.co/docs/optimum/bettertransformer/overview"),p(Me,"rel","nofollow"),p(Ue,"href","https://pytorch.org/docs/stable/nested.html"),p(Ue,"rel","nofollow"),p(Te,"href","https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2"),p(Te,"rel","nofollow"),p(Pe,"href","https://github.com/huggingface/optimum"),p(Pe,"rel","nofollow"),p(lt,"href","/docs/transformers/v4.31.0/en/main_classes/model#transformers.PreTrainedModel.to_bettertransformer"),p(rt,"href","/docs/transformers/v4.31.0/en/main_classes/model#transformers.PreTrainedModel.reverse_bettertransformer"),p(Je,"href","https://huggingface.co/docs/optimum/bettertransformer/overview#supported-models"),p(Je,"rel","nofollow"),p(ee,"id","bitsandbytes-integration-for-fp4-mixedprecision-inference"),p(ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(ee,"href","#bitsandbytes-integration-for-fp4-mixedprecision-inference"),p(j,"class","relative group"),p(le,"id","requirements"),p(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(le,"href","#requirements"),p(N,"class","relative group"),p(ie,"id","running-fp4-models-single-gpu-setup-quickstart"),p(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(ie,"href","#running-fp4-models-single-gpu-setup-quickstart"),p(q,"class","relative group"),p(ne,"id","running-fp4-models-multi-gpu-setup"),p(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(ne,"href","#running-fp4-models-multi-gpu-setup"),p(V,"class","relative group"),p(fe,"id","advanced-usage"),p(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(fe,"href","#advanced-usage"),p(B,"class","relative group"),p(ft,"href","main_classes/quantization"),p(me,"id","bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),p(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(me,"href","#bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),p(X,"class","relative group"),p(Ne,"href","https://arxiv.org/abs/2208.07339"),p(Ne,"rel","nofollow"),cr(ut.src,La="https://s3.amazonaws.com/moonup/production/uploads/1659861207959-62441d1d9fdefb55a0b7d12c.png")||p(ut,"src",La),p(ut,"alt","HFxbitsandbytes.png"),p(qe,"href","https://arxiv.org/abs/2208.07339"),p(qe,"rel","nofollow"),p(Ve,"href","https://huggingface.co/blog/hf-bitsandbytes-integration"),p(Ve,"rel","nofollow"),cr(ht.src,Ka="https://s3.amazonaws.com/moonup/production/uploads/1660567469965-62441d1d9fdefb55a0b7d12c.gif")||p(ht,"src",Ka),p(ht,"alt","MixedInt8.gif"),p(dt,"href","#colab-demos"),p(he,"id","requirements"),p(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(he,"href","#requirements"),p(Y,"class","relative group"),p(ve,"id","running-mixedint8-models-single-gpu-setup"),p(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(ve,"href","#running-mixedint8-models-single-gpu-setup"),p(O,"class","relative group"),p(ge,"id","running-mixedint8-models-multi-gpu-setup"),p(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(ge,"href","#running-mixedint8-models-multi-gpu-setup"),p(z,"class","relative group"),p(_e,"id","colab-demos"),p(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(_e,"href","#colab-demos"),p(Q,"class","relative group"),cr($t.src,ei="https://colab.research.google.com/assets/colab-badge.svg")||p($t,"src",ei),p($t,"alt","Open In Colab: T5-11b demo"),p(He,"href","https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing"),p(He,"rel","nofollow"),cr(Tt.src,ti="https://colab.research.google.com/assets/colab-badge.svg")||p(Tt,"src",ti),p(Tt,"alt","Open In Colab: BLOOM-3b demo"),p(Le,"href","https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing"),p(Le,"rel","nofollow")},m(e,n){t(document.head,c),f(e,M,n),f(e,_,n),t(_,w),t(w,Rt),b(we,Rt,null),t(_,hr),t(_,jt),t(jt,dr),f(e,Do,n),f(e,G,n),t(G,br),t(G,tt),t(tt,vr),t(G,yr),t(G,ot),t(ot,gr),t(G,_r),f(e,Ho,n),f(e,R,n),t(R,H),t(H,Nt),b(Ee,Nt,null),t(R,wr),t(R,qt),t(qt,Er),f(e,Lo,n),f(e,k,n),t(k,$r),t(k,$e),t($e,Vt),t(Vt,Mr),t(k,Ur),t(k,Me),t(Me,Tr),t(k,Pr),f(e,Ko,n),f(e,J,n),t(J,Gr),t(J,Ue),t(Ue,kr),t(J,Jr),t(J,Te),t(Te,xr),t(J,Cr),f(e,el,n),f(e,x,n),t(x,Ir),t(x,Pe),t(Pe,Bt),t(Bt,Wr),t(x,Fr),t(x,lt),t(lt,Ar),t(x,Zr),f(e,tl,n),b(Ge,e,n),f(e,ol,n),f(e,L,n),t(L,Rr),t(L,rt),t(rt,jr),t(L,Nr),f(e,ll,n),b(ke,e,n),f(e,rl,n),f(e,K,n),t(K,qr),t(K,Je),t(Je,Vr),t(K,Br),f(e,sl,n),f(e,j,n),t(j,ee),t(ee,Xt),b(xe,Xt,null),t(j,Xr),t(j,st),t(st,Yt),t(Yt,Yr),t(st,Sr),f(e,al,n),f(e,te,n),t(te,Or),t(te,St),t(St,zr),t(te,Qr),f(e,il,n),b(oe,e,n),f(e,nl,n),f(e,N,n),t(N,le),t(le,Ot),b(Ce,Ot,null),t(N,Dr),t(N,zt),t(zt,Hr),f(e,fl,n),f(e,C,n),t(C,Qt),t(Qt,re),t(re,Lr),t(re,Dt),t(Dt,Kr),t(re,es),t(re,Ht),t(Ht,ts),t(C,os),t(C,Lt),t(Lt,se),t(se,ls),t(se,Kt),t(Kt,rs),t(se,ss),t(se,eo),t(eo,as),t(C,is),t(C,to),t(to,ae),t(ae,ns),t(ae,oo),t(oo,fs),t(ae,ps),t(ae,lo),t(lo,ms),f(e,pl,n),f(e,q,n),t(q,ie),t(ie,ro),b(Ie,ro,null),t(q,us),t(q,so),t(so,cs),f(e,ml,n),f(e,at,n),t(at,hs),f(e,ul,n),b(We,e,n),f(e,cl,n),f(e,I,n),t(I,ds),t(I,ao),t(ao,bs),t(I,vs),t(I,io),t(io,ys),t(I,gs),f(e,hl,n),f(e,V,n),t(V,ne),t(ne,no),b(Fe,no,null),t(V,_s),t(V,fo),t(fo,ws),f(e,dl,n),f(e,it,n),t(it,Es),f(e,bl,n),b(Ae,e,n),f(e,vl,n),f(e,W,n),t(W,$s),t(W,po),t(po,Ms),t(W,Us),t(W,mo),t(mo,Ts),t(W,Ps),f(e,yl,n),b(Ze,e,n),f(e,gl,n),f(e,nt,n),t(nt,Gs),f(e,_l,n),f(e,B,n),t(B,fe),t(fe,uo),b(Re,uo,null),t(B,ks),t(B,co),t(co,Js),f(e,wl,n),f(e,pe,n),t(pe,xs),t(pe,ft),t(ft,Cs),t(pe,Is),f(e,El,n),f(e,X,n),t(X,me),t(me,ho),b(je,ho,null),t(X,Ws),t(X,pt),t(pt,bo),t(bo,Fs),t(pt,As),f(e,$l,n),b(ue,e,n),f(e,Ml,n),f(e,E,n),t(E,Zs),t(E,Ne),t(Ne,vo),t(vo,Rs),t(E,js),t(E,yo),t(yo,Ns),t(E,qs),t(E,go),t(go,Vs),t(E,Bs),t(E,_o),t(_o,Xs),t(E,Ys),t(E,wo),t(wo,Ss),t(E,Os),f(e,Ul,n),f(e,mt,n),t(mt,ut),f(e,Tl,n),f(e,F,n),t(F,zs),t(F,qe),t(qe,Qs),t(F,Ds),t(F,Ve),t(Ve,Hs),t(F,Ls),f(e,Pl,n),f(e,ct,n),t(ct,ht),f(e,Gl,n),f(e,ce,n),t(ce,Ks),t(ce,dt),t(dt,ea),t(ce,ta),f(e,kl,n),f(e,Y,n),t(Y,he),t(he,Eo),b(Be,Eo,null),t(Y,oa),t(Y,$o),t($o,la),f(e,Jl,n),f(e,A,n),t(A,S),t(S,ra),t(S,Mo),t(Mo,sa),t(S,aa),t(S,Uo),t(Uo,ia),t(S,na),t(A,fa),t(A,de),t(de,pa),t(de,To),t(To,ma),t(de,ua),t(de,Po),t(Po,ca),t(A,ha),t(A,be),t(be,da),t(be,Go),t(Go,ba),t(be,va),t(be,ko),t(ko,ya),f(e,xl,n),f(e,O,n),t(O,ve),t(ve,Jo),b(Xe,Jo,null),t(O,ga),t(O,xo),t(xo,_a),f(e,Cl,n),f(e,bt,n),t(bt,wa),f(e,Il,n),b(Ye,e,n),f(e,Wl,n),f(e,vt,n),t(vt,Ea),f(e,Fl,n),f(e,ye,n),t(ye,$),t($,$a),t($,Co),t(Co,Ma),t($,Ua),t($,Io),t(Io,Ta),t($,Pa),t($,Wo),t(Wo,Ga),t($,ka),t($,Fo),t(Fo,Ja),t($,xa),t($,Ao),t(Ao,Ca),t($,Ia),t(ye,Wa),t(ye,Zo),t(Zo,Fa),f(e,Al,n),f(e,yt,n),t(yt,Aa),f(e,Zl,n),b(Se,e,n),f(e,Rl,n),f(e,z,n),t(z,ge),t(ge,Ro),b(Oe,Ro,null),t(z,Za),t(z,jo),t(jo,Ra),f(e,jl,n),f(e,gt,n),t(gt,ja),f(e,Nl,n),b(ze,e,n),f(e,ql,n),f(e,Z,n),t(Z,Na),t(Z,No),t(No,qa),t(Z,Va),t(Z,qo),t(qo,Ba),t(Z,Xa),f(e,Vl,n),b(Qe,e,n),f(e,Bl,n),f(e,_t,n),t(_t,Ya),f(e,Xl,n),f(e,Q,n),t(Q,_e),t(_e,Vo),b(De,Vo,null),t(Q,Sa),t(Q,Bo),t(Bo,Oa),f(e,Yl,n),f(e,wt,n),t(wt,za),f(e,Sl,n),f(e,Et,n),t(Et,He),t(He,$t),f(e,Ol,n),f(e,Mt,n),t(Mt,Qa),f(e,zl,n),f(e,Ut,n),t(Ut,Le),t(Le,Tt),Ql=!0},p(e,[n]){const Ke={};n&2&&(Ke.$$scope={dirty:n,ctx:e}),oe.$set(Ke);const Xo={};n&2&&(Xo.$$scope={dirty:n,ctx:e}),ue.$set(Xo)},i(e){Ql||(v(we.$$.fragment,e),v(Ee.$$.fragment,e),v(Ge.$$.fragment,e),v(ke.$$.fragment,e),v(xe.$$.fragment,e),v(oe.$$.fragment,e),v(Ce.$$.fragment,e),v(Ie.$$.fragment,e),v(We.$$.fragment,e),v(Fe.$$.fragment,e),v(Ae.$$.fragment,e),v(Ze.$$.fragment,e),v(Re.$$.fragment,e),v(je.$$.fragment,e),v(ue.$$.fragment,e),v(Be.$$.fragment,e),v(Xe.$$.fragment,e),v(Ye.$$.fragment,e),v(Se.$$.fragment,e),v(Oe.$$.fragment,e),v(ze.$$.fragment,e),v(Qe.$$.fragment,e),v(De.$$.fragment,e),Ql=!0)},o(e){y(we.$$.fragment,e),y(Ee.$$.fragment,e),y(Ge.$$.fragment,e),y(ke.$$.fragment,e),y(xe.$$.fragment,e),y(oe.$$.fragment,e),y(Ce.$$.fragment,e),y(Ie.$$.fragment,e),y(We.$$.fragment,e),y(Fe.$$.fragment,e),y(Ae.$$.fragment,e),y(Ze.$$.fragment,e),y(Re.$$.fragment,e),y(je.$$.fragment,e),y(ue.$$.fragment,e),y(Be.$$.fragment,e),y(Xe.$$.fragment,e),y(Ye.$$.fragment,e),y(Se.$$.fragment,e),y(Oe.$$.fragment,e),y(ze.$$.fragment,e),y(Qe.$$.fragment,e),y(De.$$.fragment,e),Ql=!1},d(e){o(c),e&&o(M),e&&o(_),g(we),e&&o(Do),e&&o(G),e&&o(Ho),e&&o(R),g(Ee),e&&o(Lo),e&&o(k),e&&o(Ko),e&&o(J),e&&o(el),e&&o(x),e&&o(tl),g(Ge,e),e&&o(ol),e&&o(L),e&&o(ll),g(ke,e),e&&o(rl),e&&o(K),e&&o(sl),e&&o(j),g(xe),e&&o(al),e&&o(te),e&&o(il),g(oe,e),e&&o(nl),e&&o(N),g(Ce),e&&o(fl),e&&o(C),e&&o(pl),e&&o(q),g(Ie),e&&o(ml),e&&o(at),e&&o(ul),g(We,e),e&&o(cl),e&&o(I),e&&o(hl),e&&o(V),g(Fe),e&&o(dl),e&&o(it),e&&o(bl),g(Ae,e),e&&o(vl),e&&o(W),e&&o(yl),g(Ze,e),e&&o(gl),e&&o(nt),e&&o(_l),e&&o(B),g(Re),e&&o(wl),e&&o(pe),e&&o(El),e&&o(X),g(je),e&&o($l),g(ue,e),e&&o(Ml),e&&o(E),e&&o(Ul),e&&o(mt),e&&o(Tl),e&&o(F),e&&o(Pl),e&&o(ct),e&&o(Gl),e&&o(ce),e&&o(kl),e&&o(Y),g(Be),e&&o(Jl),e&&o(A),e&&o(xl),e&&o(O),g(Xe),e&&o(Cl),e&&o(bt),e&&o(Il),g(Ye,e),e&&o(Wl),e&&o(vt),e&&o(Fl),e&&o(ye),e&&o(Al),e&&o(yt),e&&o(Zl),g(Se,e),e&&o(Rl),e&&o(z),g(Oe),e&&o(jl),e&&o(gt),e&&o(Nl),g(ze,e),e&&o(ql),e&&o(Z),e&&o(Vl),g(Qe,e),e&&o(Bl),e&&o(_t),e&&o(Xl),e&&o(Q),g(De),e&&o(Yl),e&&o(wt),e&&o(Sl),e&&o(Et),e&&o(Ol),e&&o(Mt),e&&o(zl),e&&o(Ut)}}}const sf={local:"efficient-inference-on-a-single-gpu",sections:[{local:"better-transformer-pytorchnative-transformer-fastpath",title:"Better Transformer: PyTorch-native transformer fastpath"},{local:"bitsandbytes-integration-for-fp4-mixedprecision-inference",sections:[{local:"requirements",title:"Requirements"},{local:"running-fp4-models-single-gpu-setup-quickstart",title:"Running FP4 models - single GPU setup - Quickstart"},{local:"running-fp4-models-multi-gpu-setup",title:"Running FP4 models - multi GPU setup"},{local:"advanced-usage",title:"Advanced usage"}],title:"`bitsandbytes` integration for FP4 mixed-precision inference"},{local:"bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition",sections:[{local:"requirements",title:"Requirements"},{local:"running-mixedint8-models-single-gpu-setup",title:"Running mixed-Int8 models - single GPU setup"},{local:"running-mixedint8-models-multi-gpu-setup",title:"Running mixed-int8 models - multi GPU setup"},{local:"colab-demos",title:"Colab demos"}],title:"`bitsandbytes` integration for Int8 mixed-precision matrix decomposition"}],title:"Efficient Inference on a Single GPU"};function af(et){return tf(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class uf extends Hn{constructor(c){super();Ln(this,c,af,rf,Kn,{})}}export{uf as default,sf as metadata};
