import{S as Xu,i as Fu,s as Vu,e as a,k as h,w as u,t as r,M as Iu,c as s,d as o,m as c,a as n,x as p,h as l,b as f,G as t,g as d,y as m,q as b,o as y,B as v,v as Ru}from"../../chunks/vendor-hf-doc-builder.js";import{T as sc}from"../../chunks/Tip-hf-doc-builder.js";import{D as ke}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Z}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as T}from"../../chunks/IconCopyLink-hf-doc-builder.js";function xu(Je){let _,k;return{c(){_=a("p"),k=r("Note that once a model has been loaded in 4-bit it is currently not possible to push the quantized weights on the Hub. Note also that you cannot train 4-bit weights as this is not supported yet. However you can use 4-bit models to train extra parameters, this will be covered in the next section.")},l(w){_=s(w,"P",{});var E=n(_);k=l(E,"Note that once a model has been loaded in 4-bit it is currently not possible to push the quantized weights on the Hub. Note also that you cannot train 4-bit weights as this is not supported yet. However you can use 4-bit models to train extra parameters, this will be covered in the next section."),E.forEach(o)},m(w,E){d(w,_,E),t(_,k)},d(w){w&&o(_)}}}function Du(Je){let _,k,w,E,te,z,mt,oe,ae,Ce,N,A,se,V;return{c(){_=a("p"),k=r("Note that once a model has been loaded in 8-bit it is currently not possible to push the quantized weights on the Hub except if you use the latest "),w=a("code"),E=r("transformers"),te=r(" and "),z=a("code"),mt=r("bitsandbytes"),oe=r(`. Note also that you cannot train 8-bit weights as this is not supported yet. However you can use 8-bit models to train extra parameters, this will be covered in the next section.
Note also that `),ae=a("code"),Ce=r("device_map"),N=r(" is optional but setting "),A=a("code"),se=r("device_map = 'auto'"),V=r(" is prefered for inference as it will dispatch efficiently the model on the available ressources.")},l(ne){_=s(ne,"P",{});var M=n(_);k=l(M,"Note that once a model has been loaded in 8-bit it is currently not possible to push the quantized weights on the Hub except if you use the latest "),w=s(M,"CODE",{});var Ae=n(w);E=l(Ae,"transformers"),Ae.forEach(o),te=l(M," and "),z=s(M,"CODE",{});var uo=n(z);mt=l(uo,"bitsandbytes"),uo.forEach(o),oe=l(M,`. Note also that you cannot train 8-bit weights as this is not supported yet. However you can use 8-bit models to train extra parameters, this will be covered in the next section.
Note also that `),ae=s(M,"CODE",{});var po=n(ae);Ce=l(po,"device_map"),po.forEach(o),N=l(M," is optional but setting "),A=s(M,"CODE",{});var bt=n(A);se=l(bt,"device_map = 'auto'"),bt.forEach(o),V=l(M," is prefered for inference as it will dispatch efficiently the model on the available ressources."),M.forEach(o)},m(ne,M){d(ne,_,M),t(_,k),t(_,w),t(w,E),t(_,te),t(_,z),t(z,mt),t(_,oe),t(_,ae),t(ae,Ce),t(_,N),t(_,A),t(A,se),t(_,V)},d(ne){ne&&o(_)}}}function Qu(Je){let _,k;return{c(){_=a("p"),k=r("Pushing 8bit models on the Hub is strongely encouraged for large models. This will allow the community to benefit from the memory footprint reduction and loading for example large models on a Google Colab.")},l(w){_=s(w,"P",{});var E=n(_);k=l(E,"Pushing 8bit models on the Hub is strongely encouraged for large models. This will allow the community to benefit from the memory footprint reduction and loading for example large models on a Google Colab."),E.forEach(o)},m(w,E){d(w,_,E),t(_,k)},d(w){w&&o(_)}}}function Gu(Je){let _,k,w,E,te,z,mt,oe,ae,Ce,N,A,se,V,ne,M,Ae,uo,po,bt,W,qr,Xo,Zr,Wr,Fo,jr,Pr,Vo,Xr,Fr,Rs,I,Vr,yt,Ir,Rr,vt,xr,Dr,xs,R,Qr,Io,Gr,Sr,Ro,Or,Yr,Ds,Be,Hr,_t,Lr,Kr,Qs,Ue,el,xo,tl,ol,Gs,re,ze,Do,gt,al,Qo,sl,Ss,le,Ne,Go,wt,nl,So,rl,Os,mo,ll,Ys,x,Oo,qe,il,Yo,dl,hl,Ho,cl,fl,Lo,Ze,ul,Ko,pl,ml,ea,bl,yl,ta,We,vl,oa,_l,gl,aa,wl,Hs,ie,je,sa,$t,$l,na,El,Ls,D,Ml,ra,Tl,kl,la,Jl,Cl,Ks,Et,en,Pe,tn,de,Xe,ia,Mt,Al,da,Bl,on,Q,Ul,ha,zl,Nl,ca,ql,Zl,an,Tt,sn,Fe,Wl,bo,jl,Pl,nn,Ve,Xl,fa,Fl,Vl,rn,kt,ln,yo,Il,dn,Ie,hn,he,Re,ua,Jt,Rl,pa,xl,cn,vo,Dl,fn,ce,xe,ma,Ct,Ql,ba,Gl,un,G,Sl,ya,Ol,Yl,va,Hl,Ll,pn,At,mn,fe,De,_a,Bt,Kl,ga,ei,bn,_o,ti,yn,Ut,vn,ue,Qe,wa,zt,oi,$a,ai,_n,go,si,gn,Nt,wn,pe,Ge,Ea,qt,ni,Ma,ri,$n,j,li,Ta,ii,di,ka,hi,ci,Ja,fi,ui,En,Zt,Mn,Se,Tn,me,Oe,Ca,Wt,pi,Aa,mi,kn,S,bi,Ba,yi,vi,Ua,_i,gi,Jn,jt,Cn,J,wi,za,$i,Ei,Na,Mi,Ti,qa,ki,Ji,Za,Ci,Ai,Wa,Bi,Ui,An,be,Ye,ja,Pt,zi,Pa,Ni,Bn,wo,qi,Un,ye,He,Xa,Xt,Zi,Le,Wi,Fa,ji,Pi,Va,Xi,zn,B,Fi,Ia,Vi,Ii,Ra,Ri,xi,xa,Di,Qi,Da,Gi,Si,Nn,U,Oi,Qa,Yi,Hi,Ga,Li,Ki,Sa,ed,td,Oa,od,ad,qn,Ft,Zn,O,sd,Ya,nd,rd,Ha,ld,id,Wn,Vt,jn,$o,dd,Pn,It,Xn,Eo,hd,Fn,ve,Ke,La,Rt,cd,Mo,fd,Ka,ud,Vn,Y,pd,es,md,bd,ts,yd,vd,In,xt,Rn,_e,et,os,Dt,_d,as,gd,xn,ge,wd,ss,$d,Ed,ns,Md,Dn,Qt,Qn,we,tt,rs,Gt,Td,ls,kd,Gn,P,Jd,is,Cd,Ad,ds,Bd,Ud,St,hs,zd,Nd,Sn,C,qd,cs,Zd,Wd,fs,jd,Pd,us,Xd,Fd,ps,Vd,Id,ms,Rd,xd,On,$e,ot,bs,Ot,Dd,ys,Qd,Yn,g,Yt,Gd,Ht,Sd,vs,Od,Yd,Hd,Ee,Ld,_s,Kd,eh,gs,th,oh,ah,q,sh,ws,nh,rh,$s,lh,ih,Es,dh,hh,Ms,ch,fh,uh,at,Lt,ph,Kt,mh,To,bh,yh,vh,st,eo,_h,Me,gh,Ts,wh,$h,ks,Eh,Mh,Th,nt,to,kh,Js,Jh,Ch,rt,oo,Ah,ao,Bh,Cs,Uh,zh,Nh,lt,so,qh,no,Zh,As,Wh,jh,Ph,it,ro,Xh,Bs,Fh,Vh,dt,lo,Ih,Us,Rh,xh,ht,io,Dh,zs,Qh,Hn,Te,ct,Ns,ho,Gh,ko,Sh,qs,Oh,Ln,H,Yh,co,Hh,Lh,Zs,Kh,ec,Kn;return z=new T({}),V=new T({}),gt=new T({}),wt=new T({}),$t=new T({}),Et=new Z({props:{code:"JTIzJTIwcGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUp",highlighted:`<span class="hljs-comment"># pip install transformers accelerate bitsandbytes</span>
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`}}),Pe=new sc({props:{warning:!0,$$slots:{default:[xu]},$$scope:{ctx:Je}}}),Mt=new T({}),Tt=new Z({props:{code:"JTIzJTIwcGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUp",highlighted:`<span class="hljs-comment"># pip install transformers accelerate bitsandbytes</span>
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),kt=new Z({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())'}}),Ie=new sc({props:{warning:!0,$$slots:{default:[Du]},$$scope:{ctx:Je}}}),Jt=new T({}),Ct=new T({}),At=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`}}),Bt=new T({}),Ut=new Z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW5mNF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBbW9kZWxfbmY0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRG5mNF9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

nf4_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

model_nf4 = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=nf4_config)`}}),zt=new T({}),Nt=new Z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWRvdWJsZV9xdWFudF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBKSUwQSUwQW1vZGVsX2RvdWJsZV9xdWFudCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCUyQyUyMHF1YW50aXphdGlvbl9jb25maWclM0Rkb3VibGVfcXVhbnRfY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

double_quant_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

model_double_quant = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=double_quant_config)`}}),qt=new T({}),Zt=new Z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmJpZ3NjaWVuY2UlMkZibG9vbS01NjBtJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmJpZ3NjaWVuY2UlMkZibG9vbS01NjBtJTIyKSUwQSUwQW1vZGVsLnB1c2hfdG9faHViKCUyMmJsb29tLTU2MG0tOGJpdCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>)

model.push_to_hub(<span class="hljs-string">&quot;bloom-560m-8bit&quot;</span>)`}}),Se=new sc({props:{warning:!0,$$slots:{default:[Qu]},$$scope:{ctx:Je}}}),Wt=new T({}),jt=new Z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMiU3QnlvdXJfdXNlcm5hbWUlN0QlMkZibG9vbS01NjBtLThiaXQlMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/bloom-560m-8bit&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`}}),Pt=new T({}),Xt=new T({}),Ft=new Z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcobGxtX2ludDhfZW5hYmxlX2ZwMzJfY3B1X29mZmxvYWQlM0RUcnVlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=<span class="hljs-literal">True</span>)`}}),Vt=new Z({props:{code:"ZGV2aWNlX21hcCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLndvcmRfZW1iZWRkaW5ncyUyMiUzQSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci53b3JkX2VtYmVkZGluZ3NfbGF5ZXJub3JtJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMmxtX2hlYWQlMjIlM0ElMjAlMjJjcHUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci5oJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLmxuX2YlMjIlM0ElMjAwJTJDJTBBJTdE",highlighted:`device_map = {
    <span class="hljs-string">&quot;transformer.word_embeddings&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.word_embeddings_layernorm&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;lm_head&quot;</span>: <span class="hljs-string">&quot;cpu&quot;</span>,
    <span class="hljs-string">&quot;transformer.h&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.ln_f&quot;</span>: <span class="hljs-number">0</span>,
}`}}),It=new Z({props:{code:"bW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>,
    device_map=device_map,
    quantization_config=quantization_config,
)`}}),Rt=new T({}),xt=new Z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbGxtX2ludDhfdGhyZXNob2xkJTNEMTAlMkMlMEEpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBtb2RlbF9pZCUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0RkZXZpY2VfbWFwJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEEpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)
tokenizer = AutoTokenizer.from_pretrained(model_id)`}}),Dt=new T({}),Qt=new Z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbGxtX2ludDhfc2tpcF9tb2R1bGVzJTNEJTVCJTIybG1faGVhZCUyMiU1RCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_skip_modules=[<span class="hljs-string">&quot;lm_head&quot;</span>],
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)
tokenizer = AutoTokenizer.from_pretrained(model_id)`}}),Gt=new T({}),Ot=new T({}),Yt=new ke({props:{name:"class transformers.BitsAndBytesConfig",anchor:"transformers.BitsAndBytesConfig",parameters:[{name:"load_in_8bit",val:" = False"},{name:"load_in_4bit",val:" = False"},{name:"llm_int8_threshold",val:" = 6.0"},{name:"llm_int8_skip_modules",val:" = None"},{name:"llm_int8_enable_fp32_cpu_offload",val:" = False"},{name:"llm_int8_has_fp16_weight",val:" = False"},{name:"bnb_4bit_compute_dtype",val:" = None"},{name:"bnb_4bit_quant_type",val:" = 'fp4'"},{name:"bnb_4bit_use_double_quant",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BitsAndBytesConfig.load_in_8bit",description:`<strong>load_in_8bit</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag is used to enable 8-bit quantization with LLM.int8().`,name:"load_in_8bit"},{anchor:"transformers.BitsAndBytesConfig.load_in_4bit",description:`<strong>load_in_4bit</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag is used to enable 4-bit quantization by replacing the Linear layers with FP4/NF4 layers from
<code>bitsandbytes</code>.`,name:"load_in_4bit"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_threshold",description:`<strong>llm_int8_threshold</strong> (<code>float</code>, <em>optional</em>, defaults to 6) &#x2014;
This corresponds to the outlier threshold for outlier detection as described in <code>LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale</code> paper: <a href="https://arxiv.org/abs/2208.07339" rel="nofollow">https://arxiv.org/abs/2208.07339</a> Any hidden states value
that is above this threshold will be considered an outlier and the operation on those values will be done
in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but
there are some exceptional systematic outliers that are very differently distributed for large models.
These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of
magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6,
but a lower threshold might be needed for more unstable models (small models, fine-tuning).`,name:"llm_int8_threshold"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_skip_modules",description:`<strong>llm_int8_skip_modules</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
An explicit list of the modules that we do not want to convert in 8-bit. This is useful for models such as
Jukebox that has several heads in different places and not necessarily at the last position. For example
for <code>CausalLM</code> models, the last <code>lm_head</code> is kept in its original <code>dtype</code>.`,name:"llm_int8_skip_modules"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_enable_fp32_cpu_offload",description:`<strong>llm_int8_enable_fp32_cpu_offload</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag is used for advanced use cases and users that are aware of this feature. If you want to split
your model in different parts and run some parts in int8 on GPU and some parts in fp32 on CPU, you can use
this flag. This is useful for offloading large models such as <code>google/flan-t5-xxl</code>. Note that the int8
operations will not be run on CPU.`,name:"llm_int8_enable_fp32_cpu_offload"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_has_fp16_weight",description:`<strong>llm_int8_has_fp16_weight</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag runs LLM.int8() with 16-bit main weights. This is useful for fine-tuning as the weights do not
have to be converted back and forth for the backward pass.`,name:"llm_int8_has_fp16_weight"},{anchor:"transformers.BitsAndBytesConfig.bnb_4bit_compute_dtype",description:`<strong>bnb_4bit_compute_dtype</strong> (<code>torch.dtype</code> or str, <em>optional</em>, defaults to <code>torch.float32</code>) &#x2014;
This sets the computational type which might be different than the input time. For example, inputs might be
fp32, but computation can be set to bf16 for speedups.`,name:"bnb_4bit_compute_dtype"},{anchor:"transformers.BitsAndBytesConfig.bnb_4bit_quant_type",description:`<strong>bnb_4bit_quant_type</strong> (<code>str</code>, {fp4, nf4}, defaults to <code>fp4</code>) &#x2014;
This sets the quantization data type in the bnb.nn.Linear4Bit layers. Options are FP4 and NF4 data types
which are specified by <code>fp4</code> or <code>nf4</code>.`,name:"bnb_4bit_quant_type"},{anchor:"transformers.BitsAndBytesConfig.bnb_4bit_use_double_quant",description:`<strong>bnb_4bit_use_double_quant</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag is used for nested quantization where the quantization constants from the first quantization are
quantized again.`,name:"bnb_4bit_use_double_quant"},{anchor:"transformers.BitsAndBytesConfig.kwargs",description:`<strong>kwargs</strong> (<code>Dict[str, Any]</code>, <em>optional</em>) &#x2014;
Additional parameters from which to initialize the configuration object.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/utils/quantization_config.py#L37"}}),Lt=new ke({props:{name:"from_dict",anchor:"transformers.BitsAndBytesConfig.from_dict",parameters:[{name:"config_dict",val:""},{name:"return_unused_kwargs",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BitsAndBytesConfig.from_dict.config_dict",description:`<strong>config_dict</strong> (<code>Dict[str, Any]</code>) &#x2014;
Dictionary that will be used to instantiate the configuration object.`,name:"config_dict"},{anchor:"transformers.BitsAndBytesConfig.from_dict.return_unused_kwargs",description:`<strong>return_unused_kwargs</strong> (<code>bool</code>) &#x2014;
Whether or not to return a list of unused keyword arguments. Used for <code>from_pretrained</code> method in
<code>PreTrainedModel</code>.`,name:"return_unused_kwargs"},{anchor:"transformers.BitsAndBytesConfig.from_dict.kwargs",description:`<strong>kwargs</strong> (<code>Dict[str, Any]</code>) &#x2014;
Additional parameters from which to initialize the configuration object.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/utils/quantization_config.py#L171",returnDescription:`
<p>The configuration object instantiated from those parameters.</p>
`,returnType:`
<p><a
  href="/docs/transformers/v4.31.0/en/main_classes/quantization#transformers.BitsAndBytesConfig"
>BitsAndBytesConfig</a></p>
`}}),eo=new ke({props:{name:"is_quantizable",anchor:"transformers.BitsAndBytesConfig.is_quantizable",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/utils/quantization_config.py#L151"}}),to=new ke({props:{name:"post_init",anchor:"transformers.BitsAndBytesConfig.post_init",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/utils/quantization_config.py#L120"}}),oo=new ke({props:{name:"quantization_method",anchor:"transformers.BitsAndBytesConfig.quantization_method",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/utils/quantization_config.py#L157"}}),so=new ke({props:{name:"to_dict",anchor:"transformers.BitsAndBytesConfig.to_dict",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/utils/quantization_config.py#L221"}}),ro=new ke({props:{name:"to_diff_dict",anchor:"transformers.BitsAndBytesConfig.to_diff_dict",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/utils/quantization_config.py#L253",returnDescription:`
<p>Dictionary of all the attributes that make up this configuration instance,</p>
`,returnType:`
<p><code>Dict[str, Any]</code></p>
`}}),lo=new ke({props:{name:"to_json_file",anchor:"transformers.BitsAndBytesConfig.to_json_file",parameters:[{name:"json_file_path",val:": typing.Union[str, os.PathLike]"}],parametersDescription:[{anchor:"transformers.BitsAndBytesConfig.to_json_file.json_file_path",description:`<strong>json_file_path</strong> (<code>str</code> or <code>os.PathLike</code>) &#x2014;
Path to the JSON file in which this configuration instance&#x2019;s parameters will be saved.`,name:"json_file_path"},{anchor:"transformers.BitsAndBytesConfig.to_json_file.use_diff",description:`<strong>use_diff</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If set to <code>True</code>, only the difference between the config instance and the default
<code>BitsAndBytesConfig()</code> is serialized to JSON file.`,name:"use_diff"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/utils/quantization_config.py#L204"}}),io=new ke({props:{name:"to_json_string",anchor:"transformers.BitsAndBytesConfig.to_json_string",parameters:[{name:"use_diff",val:": bool = True"}],parametersDescription:[{anchor:"transformers.BitsAndBytesConfig.to_json_string.use_diff",description:`<strong>use_diff</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If set to <code>True</code>, only the difference between the config instance and the default <code>PretrainedConfig()</code>
is serialized to JSON string.`,name:"use_diff"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/utils/quantization_config.py#L235",returnDescription:`
<p>String containing all the attributes that make up this configuration instance in JSON format.</p>
`,returnType:`
<p><code>str</code></p>
`}}),ho=new T({}),{c(){_=a("meta"),k=h(),w=a("h1"),E=a("a"),te=a("span"),u(z.$$.fragment),mt=h(),oe=a("span"),ae=r("Quantize \u{1F917} Transformers models"),Ce=h(),N=a("h2"),A=a("a"),se=a("span"),u(V.$$.fragment),ne=h(),M=a("span"),Ae=a("code"),uo=r("bitsandbytes"),po=r(" Integration"),bt=h(),W=a("p"),qr=r("\u{1F917} Transformers is closely integrated with most used modules on "),Xo=a("code"),Zr=r("bitsandbytes"),Wr=r(`. You can load your model in 8-bit precision with few lines of code.
This is supported by most of the GPU hardwares since the `),Fo=a("code"),jr=r("0.37.0"),Pr=r(" release of "),Vo=a("code"),Xr=r("bitsandbytes"),Fr=r("."),Rs=h(),I=a("p"),Vr=r("Learn more about the quantization method in the "),yt=a("a"),Ir=r("LLM.int8()"),Rr=r(" paper, or the "),vt=a("a"),xr=r("blogpost"),Dr=r(" about the collaboration."),xs=h(),R=a("p"),Qr=r("Since its "),Io=a("code"),Gr=r("0.39.0"),Sr=r(" release, you can load any model that supports "),Ro=a("code"),Or=r("device_map"),Yr=r(" using 4-bit quantization, leveraging FP4 data type."),Ds=h(),Be=a("p"),Hr=r("If you want to quantize your own pytorch model, check out this "),_t=a("a"),Lr=r("documentation"),Kr=r(" from \u{1F917} Accelerate library."),Qs=h(),Ue=a("p"),el=r("Here are the things you can do using "),xo=a("code"),tl=r("bitsandbytes"),ol=r(" integration"),Gs=h(),re=a("h3"),ze=a("a"),Do=a("span"),u(gt.$$.fragment),al=h(),Qo=a("span"),sl=r("FP4 quantization"),Ss=h(),le=a("h4"),Ne=a("a"),Go=a("span"),u(wt.$$.fragment),nl=h(),So=a("span"),rl=r("Requirements"),Os=h(),mo=a("p"),ll=r("Make sure that you have installed the requirements below before running any of the code snippets below."),Ys=h(),x=a("ul"),Oo=a("li"),qe=a("p"),il=r("Latest "),Yo=a("code"),dl=r("bitsandbytes"),hl=r(` library
`),Ho=a("code"),cl=r("pip install bitsandbytes>=0.39.0"),fl=h(),Lo=a("li"),Ze=a("p"),ul=r("Install latest "),Ko=a("code"),pl=r("accelerate"),ml=r(` from source
`),ea=a("code"),bl=r("pip install git+https://github.com/huggingface/accelerate.git"),yl=h(),ta=a("li"),We=a("p"),vl=r("Install latest "),oa=a("code"),_l=r("transformers"),gl=r(` from source
`),aa=a("code"),wl=r("pip install git+https://github.com/huggingface/transformers.git"),Hs=h(),ie=a("h4"),je=a("a"),sa=a("span"),u($t.$$.fragment),$l=h(),na=a("span"),El=r("Load a large model in 4bit"),Ls=h(),D=a("p"),Ml=r("By using "),ra=a("code"),Tl=r("load_in_4bit=True"),kl=r(" when calling the "),la=a("code"),Jl=r(".from_pretrained"),Cl=r(" method, you can divide your memory use by 4 (roughly)."),Ks=h(),u(Et.$$.fragment),en=h(),u(Pe.$$.fragment),tn=h(),de=a("h3"),Xe=a("a"),ia=a("span"),u(Mt.$$.fragment),Al=h(),da=a("span"),Bl=r("Load a large model in 8bit"),on=h(),Q=a("p"),Ul=r("You can load a model by roughly halving the memory requirements by using "),ha=a("code"),zl=r("load_in_8bit=True"),Nl=r(" argument when calling "),ca=a("code"),ql=r(".from_pretrained"),Zl=r(" method"),an=h(),u(Tt.$$.fragment),sn=h(),Fe=a("p"),Wl=r("Then, use your model as you would usually use a "),bo=a("a"),jl=r("PreTrainedModel"),Pl=r("."),nn=h(),Ve=a("p"),Xl=r("You can check the memory footprint of your model with "),fa=a("code"),Fl=r("get_memory_footprint"),Vl=r(" method."),rn=h(),u(kt.$$.fragment),ln=h(),yo=a("p"),Il=r("With this integration we were able to load large models on smaller devices and run them without any issue."),dn=h(),u(Ie.$$.fragment),hn=h(),he=a("h4"),Re=a("a"),ua=a("span"),u(Jt.$$.fragment),Rl=h(),pa=a("span"),xl=r("Advanced usecases"),cn=h(),vo=a("p"),Dl=r("Here we will cover some advanced usecases you can perform with FP4 quantization"),fn=h(),ce=a("h5"),xe=a("a"),ma=a("span"),u(Ct.$$.fragment),Ql=h(),ba=a("span"),Gl=r("Change the compute dtype"),un=h(),G=a("p"),Sl=r("The compute dtype is used to change the dtype that will be used during computation. For example, hidden states could be in "),ya=a("code"),Ol=r("float32"),Yl=r(" but computation can be set to bf16 for speedups. By default, the compute dtype is set to "),va=a("code"),Hl=r("float32"),Ll=r("."),pn=h(),u(At.$$.fragment),mn=h(),fe=a("h5"),De=a("a"),_a=a("span"),u(Bt.$$.fragment),Kl=h(),ga=a("span"),ei=r("Using NF4 (Normal Float 4) data type"),bn=h(),_o=a("p"),ti=r("You can also use the NF4 data type, which is a new 4bit datatype adapted for weights that have been initialized using a normal distribution. For that run:"),yn=h(),u(Ut.$$.fragment),vn=h(),ue=a("h5"),Qe=a("a"),wa=a("span"),u(zt.$$.fragment),oi=h(),$a=a("span"),ai=r("Use nested quantization for more memory efficient inference"),_n=h(),go=a("p"),si=r("We also advise users to use the nested quantization technique. This saves more memory at no additional performance - from our empirical observations, this enables fine-tuning llama-13b model on an NVIDIA-T4 16GB with a sequence length of 1024, batch size of 1 and gradient accumulation steps of 4."),gn=h(),u(Nt.$$.fragment),wn=h(),pe=a("h3"),Ge=a("a"),Ea=a("span"),u(qt.$$.fragment),ni=h(),Ma=a("span"),ri=r("Push quantized models on the \u{1F917} Hub"),$n=h(),j=a("p"),li=r("You can push a quantized model on the Hub by naively using "),Ta=a("code"),ii=r("push_to_hub"),di=r(` method. This will first push the quantization configuration file, then push the quantized model weights.
Make sure to use `),ka=a("code"),hi=r("bitsandbytes>0.37.2"),ci=r(" (at this time of writing, we tested it on "),Ja=a("code"),fi=r("bitsandbytes==0.38.0.post1"),ui=r(") to be able to use this feature."),En=h(),u(Zt.$$.fragment),Mn=h(),u(Se.$$.fragment),Tn=h(),me=a("h3"),Oe=a("a"),Ca=a("span"),u(Wt.$$.fragment),pi=h(),Aa=a("span"),mi=r("Load a quantized model from the \u{1F917} Hub"),kn=h(),S=a("p"),bi=r("You can load a quantized model from the Hub by using "),Ba=a("code"),yi=r("from_pretrained"),vi=r(" method. Make sure that the pushed weights are quantized, by checking that the attribute "),Ua=a("code"),_i=r("quantization_config"),gi=r(" is present in the model configuration object."),Jn=h(),u(jt.$$.fragment),Cn=h(),J=a("p"),wi=r("Note that in this case, you don\u2019t need to specify the arguments "),za=a("code"),$i=r("load_in_8bit=True"),Ei=r(", but you need to make sure that "),Na=a("code"),Mi=r("bitsandbytes"),Ti=r(" and "),qa=a("code"),ki=r("accelerate"),Ji=r(` are installed.
Note also that `),Za=a("code"),Ci=r("device_map"),Ai=r(" is optional but setting "),Wa=a("code"),Bi=r("device_map = 'auto'"),Ui=r(" is prefered for inference as it will dispatch efficiently the model on the available ressources."),An=h(),be=a("h3"),Ye=a("a"),ja=a("span"),u(Pt.$$.fragment),zi=h(),Pa=a("span"),Ni=r("Advanced usecases"),Bn=h(),wo=a("p"),qi=r("This section is intended to advanced users, that want to explore what it is possible to do beyond loading and running 8-bit models."),Un=h(),ye=a("h4"),He=a("a"),Xa=a("span"),u(Xt.$$.fragment),Zi=h(),Le=a("span"),Wi=r("Offload between "),Fa=a("code"),ji=r("cpu"),Pi=r(" and "),Va=a("code"),Xi=r("gpu"),zn=h(),B=a("p"),Fi=r("One of the advanced usecase of this is being able to load a model and dispatch the weights between "),Ia=a("code"),Vi=r("CPU"),Ii=r(" and "),Ra=a("code"),Ri=r("GPU"),xi=r(". Note that the weights that will be dispatched on CPU "),xa=a("strong"),Di=r("will not"),Qi=r(" be converted in 8-bit, thus kept in "),Da=a("code"),Gi=r("float32"),Si=r(". This feature is intended for users that want to fit a very large model and dispatch the model between GPU and CPU."),Nn=h(),U=a("p"),Oi=r("First, load a "),Qa=a("code"),Yi=r("BitsAndBytesConfig"),Hi=r(" from "),Ga=a("code"),Li=r("transformers"),Ki=r(" and set the attribute "),Sa=a("code"),ed=r("llm_int8_enable_fp32_cpu_offload"),td=r(" to "),Oa=a("code"),od=r("True"),ad=r(":"),qn=h(),u(Ft.$$.fragment),Zn=h(),O=a("p"),sd=r("Let\u2019s say you want to load "),Ya=a("code"),nd=r("bigscience/bloom-1b7"),rd=r(" model, and you have just enough GPU RAM to fit the entire model except the "),Ha=a("code"),ld=r("lm_head"),id=r(". Therefore write a custom device_map as follows:"),Wn=h(),u(Vt.$$.fragment),jn=h(),$o=a("p"),dd=r("And load your model as follows:"),Pn=h(),u(It.$$.fragment),Xn=h(),Eo=a("p"),hd=r("And that\u2019s it! Enjoy your model!"),Fn=h(),ve=a("h4"),Ke=a("a"),La=a("span"),u(Rt.$$.fragment),cd=h(),Mo=a("span"),fd=r("Play with "),Ka=a("code"),ud=r("llm_int8_threshold"),Vn=h(),Y=a("p"),pd=r("You can play with the "),es=a("code"),md=r("llm_int8_threshold"),bd=r(` argument to change the threshold of the outliers. An \u201Coutlier\u201D is a hidden state value that is greater than a certain threshold.
This corresponds to the outlier threshold for outlier detection as described in `),ts=a("code"),yd=r("LLM.int8()"),vd=r(` paper. Any hidden states value that is above this threshold will be considered an outlier and the operation on those values will be done in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but there are some exceptional systematic outliers that are very differently distributed for large models. These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6, but a lower threshold might be needed for more unstable models (small models, fine-tuning).
This argument can impact the inference speed of the model. We suggest to play with this parameter to find which one is the best for your usecase.`),In=h(),u(xt.$$.fragment),Rn=h(),_e=a("h4"),et=a("a"),os=a("span"),u(Dt.$$.fragment),_d=h(),as=a("span"),gd=r("Skip the conversion of some modules"),xn=h(),ge=a("p"),wd=r("Some models has several modules that needs to be not converted in 8-bit to ensure stability. For example Jukebox model has several "),ss=a("code"),$d=r("lm_head"),Ed=r(" modules that should be skipped. Play with "),ns=a("code"),Md=r("llm_int8_skip_modules"),Dn=h(),u(Qt.$$.fragment),Qn=h(),we=a("h4"),tt=a("a"),rs=a("span"),u(Gt.$$.fragment),Td=h(),ls=a("span"),kd=r("Fine-tune a model that has been loaded in 8-bit"),Gn=h(),P=a("p"),Jd=r(`With the official support of adapters in the Hugging Face ecosystem, you can fine-tune models that have been loaded in 8-bit.
This enables fine-tuning large models such as `),is=a("code"),Cd=r("flan-t5-large"),Ad=r(" or "),ds=a("code"),Bd=r("facebook/opt-6.7b"),Ud=r(" in a single google Colab. Please have a look at "),St=a("a"),hs=a("code"),zd=r("peft"),Nd=r(" library for more details."),Sn=h(),C=a("p"),qd=r("Note that you don\u2019t need to pass "),cs=a("code"),Zd=r("device_map"),Wd=r(" when loading the model for training. It will automatically load your model on your GPU. You can also set the device map to a specific device if needed (e.g. "),fs=a("code"),jd=r("cuda:0"),Pd=r(", "),us=a("code"),Xd=r("0"),Fd=r(", "),ps=a("code"),Vd=r("torch.device('cuda:0')"),Id=r("). Please note that "),ms=a("code"),Rd=r("device_map=auto"),xd=r(" should be used for inference only."),On=h(),$e=a("h3"),ot=a("a"),bs=a("span"),u(Ot.$$.fragment),Dd=h(),ys=a("span"),Qd=r("BitsAndBytesConfig"),Yn=h(),g=a("div"),u(Yt.$$.fragment),Gd=h(),Ht=a("p"),Sd=r(`This is a wrapper class about all possible attributes and features that you can play with a model that has been
loaded using `),vs=a("code"),Od=r("bitsandbytes"),Yd=r("."),Hd=h(),Ee=a("p"),Ld=r("This replaces "),_s=a("code"),Kd=r("load_in_8bit"),eh=r(" or "),gs=a("code"),th=r("load_in_4bit"),oh=r("therefore both options are mutually exclusive."),ah=h(),q=a("p"),sh=r("Currently only supports "),ws=a("code"),nh=r("LLM.int8()"),rh=r(", "),$s=a("code"),lh=r("FP4"),ih=r(", and "),Es=a("code"),dh=r("NF4"),hh=r(" quantization. If more methods are added to "),Ms=a("code"),ch=r("bitsandbytes"),fh=r(`,
then more arguments will be added to this class.`),uh=h(),at=a("div"),u(Lt.$$.fragment),ph=h(),Kt=a("p"),mh=r("Instantiates a "),To=a("a"),bh=r("BitsAndBytesConfig"),yh=r(" from a Python dictionary of parameters."),vh=h(),st=a("div"),u(eo.$$.fragment),_h=h(),Me=a("p"),gh=r("Returns "),Ts=a("code"),wh=r("True"),$h=r(" if the model is quantizable, "),ks=a("code"),Eh=r("False"),Mh=r(" otherwise."),Th=h(),nt=a("div"),u(to.$$.fragment),kh=h(),Js=a("p"),Jh=r("Safety checker that arguments are correct - also replaces some NoneType arguments with their default values."),Ch=h(),rt=a("div"),u(oo.$$.fragment),Ah=h(),ao=a("p"),Bh=r(`This method returns the quantization method used for the model. If the model is not quantizable, it returns
`),Cs=a("code"),Uh=r("None"),zh=r("."),Nh=h(),lt=a("div"),u(so.$$.fragment),qh=h(),no=a("p"),Zh=r(`Serializes this instance to a Python dictionary. Returns:
`),As=a("code"),Wh=r("Dict[str, Any]"),jh=r(": Dictionary of all the attributes that make up this configuration instance."),Ph=h(),it=a("div"),u(ro.$$.fragment),Xh=h(),Bs=a("p"),Fh=r(`Removes all attributes from config which correspond to the default config attributes for better readability and
serializes to a Python dictionary.`),Vh=h(),dt=a("div"),u(lo.$$.fragment),Ih=h(),Us=a("p"),Rh=r("Save this instance to a JSON file."),xh=h(),ht=a("div"),u(io.$$.fragment),Dh=h(),zs=a("p"),Qh=r("Serializes this instance to a JSON string."),Hn=h(),Te=a("h2"),ct=a("a"),Ns=a("span"),u(ho.$$.fragment),Gh=h(),ko=a("span"),Sh=r("Quantization with \u{1F917} "),qs=a("code"),Oh=r("optimum"),Ln=h(),H=a("p"),Yh=r("Please have a look at "),co=a("a"),Hh=r("Optimum documentation"),Lh=r(" to learn more about quantization methods that are supported by "),Zs=a("code"),Kh=r("optimum"),ec=r(" and see if these are applicable for your usecase."),this.h()},l(e){const i=Iu('[data-svelte="svelte-1phssyn"]',document.head);_=s(i,"META",{name:!0,content:!0}),i.forEach(o),k=c(e),w=s(e,"H1",{class:!0});var fo=n(w);E=s(fo,"A",{id:!0,class:!0,href:!0});var Ws=n(E);te=s(Ws,"SPAN",{});var js=n(te);p(z.$$.fragment,js),js.forEach(o),Ws.forEach(o),mt=c(fo),oe=s(fo,"SPAN",{});var nc=n(oe);ae=l(nc,"Quantize \u{1F917} Transformers models"),nc.forEach(o),fo.forEach(o),Ce=c(e),N=s(e,"H2",{class:!0});var er=n(N);A=s(er,"A",{id:!0,class:!0,href:!0});var rc=n(A);se=s(rc,"SPAN",{});var lc=n(se);p(V.$$.fragment,lc),lc.forEach(o),rc.forEach(o),ne=c(er),M=s(er,"SPAN",{});var tc=n(M);Ae=s(tc,"CODE",{});var ic=n(Ae);uo=l(ic,"bitsandbytes"),ic.forEach(o),po=l(tc," Integration"),tc.forEach(o),er.forEach(o),bt=c(e),W=s(e,"P",{});var ft=n(W);qr=l(ft,"\u{1F917} Transformers is closely integrated with most used modules on "),Xo=s(ft,"CODE",{});var dc=n(Xo);Zr=l(dc,"bitsandbytes"),dc.forEach(o),Wr=l(ft,`. You can load your model in 8-bit precision with few lines of code.
This is supported by most of the GPU hardwares since the `),Fo=s(ft,"CODE",{});var hc=n(Fo);jr=l(hc,"0.37.0"),hc.forEach(o),Pr=l(ft," release of "),Vo=s(ft,"CODE",{});var cc=n(Vo);Xr=l(cc,"bitsandbytes"),cc.forEach(o),Fr=l(ft,"."),ft.forEach(o),Rs=c(e),I=s(e,"P",{});var Jo=n(I);Vr=l(Jo,"Learn more about the quantization method in the "),yt=s(Jo,"A",{href:!0,rel:!0});var fc=n(yt);Ir=l(fc,"LLM.int8()"),fc.forEach(o),Rr=l(Jo," paper, or the "),vt=s(Jo,"A",{href:!0,rel:!0});var uc=n(vt);xr=l(uc,"blogpost"),uc.forEach(o),Dr=l(Jo," about the collaboration."),Jo.forEach(o),xs=c(e),R=s(e,"P",{});var Co=n(R);Qr=l(Co,"Since its "),Io=s(Co,"CODE",{});var pc=n(Io);Gr=l(pc,"0.39.0"),pc.forEach(o),Sr=l(Co," release, you can load any model that supports "),Ro=s(Co,"CODE",{});var mc=n(Ro);Or=l(mc,"device_map"),mc.forEach(o),Yr=l(Co," using 4-bit quantization, leveraging FP4 data type."),Co.forEach(o),Ds=c(e),Be=s(e,"P",{});var tr=n(Be);Hr=l(tr,"If you want to quantize your own pytorch model, check out this "),_t=s(tr,"A",{href:!0,rel:!0});var bc=n(_t);Lr=l(bc,"documentation"),bc.forEach(o),Kr=l(tr," from \u{1F917} Accelerate library."),tr.forEach(o),Qs=c(e),Ue=s(e,"P",{});var or=n(Ue);el=l(or,"Here are the things you can do using "),xo=s(or,"CODE",{});var yc=n(xo);tl=l(yc,"bitsandbytes"),yc.forEach(o),ol=l(or," integration"),or.forEach(o),Gs=c(e),re=s(e,"H3",{class:!0});var ar=n(re);ze=s(ar,"A",{id:!0,class:!0,href:!0});var vc=n(ze);Do=s(vc,"SPAN",{});var _c=n(Do);p(gt.$$.fragment,_c),_c.forEach(o),vc.forEach(o),al=c(ar),Qo=s(ar,"SPAN",{});var gc=n(Qo);sl=l(gc,"FP4 quantization"),gc.forEach(o),ar.forEach(o),Ss=c(e),le=s(e,"H4",{class:!0});var sr=n(le);Ne=s(sr,"A",{id:!0,class:!0,href:!0});var wc=n(Ne);Go=s(wc,"SPAN",{});var $c=n(Go);p(wt.$$.fragment,$c),$c.forEach(o),wc.forEach(o),nl=c(sr),So=s(sr,"SPAN",{});var Ec=n(So);rl=l(Ec,"Requirements"),Ec.forEach(o),sr.forEach(o),Os=c(e),mo=s(e,"P",{});var Mc=n(mo);ll=l(Mc,"Make sure that you have installed the requirements below before running any of the code snippets below."),Mc.forEach(o),Ys=c(e),x=s(e,"UL",{});var Ao=n(x);Oo=s(Ao,"LI",{});var Tc=n(Oo);qe=s(Tc,"P",{});var Ps=n(qe);il=l(Ps,"Latest "),Yo=s(Ps,"CODE",{});var kc=n(Yo);dl=l(kc,"bitsandbytes"),kc.forEach(o),hl=l(Ps,` library
`),Ho=s(Ps,"CODE",{});var Jc=n(Ho);cl=l(Jc,"pip install bitsandbytes>=0.39.0"),Jc.forEach(o),Ps.forEach(o),Tc.forEach(o),fl=c(Ao),Lo=s(Ao,"LI",{});var Cc=n(Lo);Ze=s(Cc,"P",{});var Xs=n(Ze);ul=l(Xs,"Install latest "),Ko=s(Xs,"CODE",{});var Ac=n(Ko);pl=l(Ac,"accelerate"),Ac.forEach(o),ml=l(Xs,` from source
`),ea=s(Xs,"CODE",{});var Bc=n(ea);bl=l(Bc,"pip install git+https://github.com/huggingface/accelerate.git"),Bc.forEach(o),Xs.forEach(o),Cc.forEach(o),yl=c(Ao),ta=s(Ao,"LI",{});var Uc=n(ta);We=s(Uc,"P",{});var Fs=n(We);vl=l(Fs,"Install latest "),oa=s(Fs,"CODE",{});var zc=n(oa);_l=l(zc,"transformers"),zc.forEach(o),gl=l(Fs,` from source
`),aa=s(Fs,"CODE",{});var Nc=n(aa);wl=l(Nc,"pip install git+https://github.com/huggingface/transformers.git"),Nc.forEach(o),Fs.forEach(o),Uc.forEach(o),Ao.forEach(o),Hs=c(e),ie=s(e,"H4",{class:!0});var nr=n(ie);je=s(nr,"A",{id:!0,class:!0,href:!0});var qc=n(je);sa=s(qc,"SPAN",{});var Zc=n(sa);p($t.$$.fragment,Zc),Zc.forEach(o),qc.forEach(o),$l=c(nr),na=s(nr,"SPAN",{});var Wc=n(na);El=l(Wc,"Load a large model in 4bit"),Wc.forEach(o),nr.forEach(o),Ls=c(e),D=s(e,"P",{});var Bo=n(D);Ml=l(Bo,"By using "),ra=s(Bo,"CODE",{});var jc=n(ra);Tl=l(jc,"load_in_4bit=True"),jc.forEach(o),kl=l(Bo," when calling the "),la=s(Bo,"CODE",{});var Pc=n(la);Jl=l(Pc,".from_pretrained"),Pc.forEach(o),Cl=l(Bo," method, you can divide your memory use by 4 (roughly)."),Bo.forEach(o),Ks=c(e),p(Et.$$.fragment,e),en=c(e),p(Pe.$$.fragment,e),tn=c(e),de=s(e,"H3",{class:!0});var rr=n(de);Xe=s(rr,"A",{id:!0,class:!0,href:!0});var Xc=n(Xe);ia=s(Xc,"SPAN",{});var Fc=n(ia);p(Mt.$$.fragment,Fc),Fc.forEach(o),Xc.forEach(o),Al=c(rr),da=s(rr,"SPAN",{});var Vc=n(da);Bl=l(Vc,"Load a large model in 8bit"),Vc.forEach(o),rr.forEach(o),on=c(e),Q=s(e,"P",{});var Uo=n(Q);Ul=l(Uo,"You can load a model by roughly halving the memory requirements by using "),ha=s(Uo,"CODE",{});var Ic=n(ha);zl=l(Ic,"load_in_8bit=True"),Ic.forEach(o),Nl=l(Uo," argument when calling "),ca=s(Uo,"CODE",{});var Rc=n(ca);ql=l(Rc,".from_pretrained"),Rc.forEach(o),Zl=l(Uo," method"),Uo.forEach(o),an=c(e),p(Tt.$$.fragment,e),sn=c(e),Fe=s(e,"P",{});var lr=n(Fe);Wl=l(lr,"Then, use your model as you would usually use a "),bo=s(lr,"A",{href:!0});var xc=n(bo);jl=l(xc,"PreTrainedModel"),xc.forEach(o),Pl=l(lr,"."),lr.forEach(o),nn=c(e),Ve=s(e,"P",{});var ir=n(Ve);Xl=l(ir,"You can check the memory footprint of your model with "),fa=s(ir,"CODE",{});var Dc=n(fa);Fl=l(Dc,"get_memory_footprint"),Dc.forEach(o),Vl=l(ir," method."),ir.forEach(o),rn=c(e),p(kt.$$.fragment,e),ln=c(e),yo=s(e,"P",{});var Qc=n(yo);Il=l(Qc,"With this integration we were able to load large models on smaller devices and run them without any issue."),Qc.forEach(o),dn=c(e),p(Ie.$$.fragment,e),hn=c(e),he=s(e,"H4",{class:!0});var dr=n(he);Re=s(dr,"A",{id:!0,class:!0,href:!0});var Gc=n(Re);ua=s(Gc,"SPAN",{});var Sc=n(ua);p(Jt.$$.fragment,Sc),Sc.forEach(o),Gc.forEach(o),Rl=c(dr),pa=s(dr,"SPAN",{});var Oc=n(pa);xl=l(Oc,"Advanced usecases"),Oc.forEach(o),dr.forEach(o),cn=c(e),vo=s(e,"P",{});var Yc=n(vo);Dl=l(Yc,"Here we will cover some advanced usecases you can perform with FP4 quantization"),Yc.forEach(o),fn=c(e),ce=s(e,"H5",{class:!0});var hr=n(ce);xe=s(hr,"A",{id:!0,class:!0,href:!0});var Hc=n(xe);ma=s(Hc,"SPAN",{});var Lc=n(ma);p(Ct.$$.fragment,Lc),Lc.forEach(o),Hc.forEach(o),Ql=c(hr),ba=s(hr,"SPAN",{});var Kc=n(ba);Gl=l(Kc,"Change the compute dtype"),Kc.forEach(o),hr.forEach(o),un=c(e),G=s(e,"P",{});var zo=n(G);Sl=l(zo,"The compute dtype is used to change the dtype that will be used during computation. For example, hidden states could be in "),ya=s(zo,"CODE",{});var ef=n(ya);Ol=l(ef,"float32"),ef.forEach(o),Yl=l(zo," but computation can be set to bf16 for speedups. By default, the compute dtype is set to "),va=s(zo,"CODE",{});var tf=n(va);Hl=l(tf,"float32"),tf.forEach(o),Ll=l(zo,"."),zo.forEach(o),pn=c(e),p(At.$$.fragment,e),mn=c(e),fe=s(e,"H5",{class:!0});var cr=n(fe);De=s(cr,"A",{id:!0,class:!0,href:!0});var of=n(De);_a=s(of,"SPAN",{});var af=n(_a);p(Bt.$$.fragment,af),af.forEach(o),of.forEach(o),Kl=c(cr),ga=s(cr,"SPAN",{});var sf=n(ga);ei=l(sf,"Using NF4 (Normal Float 4) data type"),sf.forEach(o),cr.forEach(o),bn=c(e),_o=s(e,"P",{});var nf=n(_o);ti=l(nf,"You can also use the NF4 data type, which is a new 4bit datatype adapted for weights that have been initialized using a normal distribution. For that run:"),nf.forEach(o),yn=c(e),p(Ut.$$.fragment,e),vn=c(e),ue=s(e,"H5",{class:!0});var fr=n(ue);Qe=s(fr,"A",{id:!0,class:!0,href:!0});var rf=n(Qe);wa=s(rf,"SPAN",{});var lf=n(wa);p(zt.$$.fragment,lf),lf.forEach(o),rf.forEach(o),oi=c(fr),$a=s(fr,"SPAN",{});var df=n($a);ai=l(df,"Use nested quantization for more memory efficient inference"),df.forEach(o),fr.forEach(o),_n=c(e),go=s(e,"P",{});var hf=n(go);si=l(hf,"We also advise users to use the nested quantization technique. This saves more memory at no additional performance - from our empirical observations, this enables fine-tuning llama-13b model on an NVIDIA-T4 16GB with a sequence length of 1024, batch size of 1 and gradient accumulation steps of 4."),hf.forEach(o),gn=c(e),p(Nt.$$.fragment,e),wn=c(e),pe=s(e,"H3",{class:!0});var ur=n(pe);Ge=s(ur,"A",{id:!0,class:!0,href:!0});var cf=n(Ge);Ea=s(cf,"SPAN",{});var ff=n(Ea);p(qt.$$.fragment,ff),ff.forEach(o),cf.forEach(o),ni=c(ur),Ma=s(ur,"SPAN",{});var uf=n(Ma);ri=l(uf,"Push quantized models on the \u{1F917} Hub"),uf.forEach(o),ur.forEach(o),$n=c(e),j=s(e,"P",{});var ut=n(j);li=l(ut,"You can push a quantized model on the Hub by naively using "),Ta=s(ut,"CODE",{});var pf=n(Ta);ii=l(pf,"push_to_hub"),pf.forEach(o),di=l(ut,` method. This will first push the quantization configuration file, then push the quantized model weights.
Make sure to use `),ka=s(ut,"CODE",{});var mf=n(ka);hi=l(mf,"bitsandbytes>0.37.2"),mf.forEach(o),ci=l(ut," (at this time of writing, we tested it on "),Ja=s(ut,"CODE",{});var bf=n(Ja);fi=l(bf,"bitsandbytes==0.38.0.post1"),bf.forEach(o),ui=l(ut,") to be able to use this feature."),ut.forEach(o),En=c(e),p(Zt.$$.fragment,e),Mn=c(e),p(Se.$$.fragment,e),Tn=c(e),me=s(e,"H3",{class:!0});var pr=n(me);Oe=s(pr,"A",{id:!0,class:!0,href:!0});var yf=n(Oe);Ca=s(yf,"SPAN",{});var vf=n(Ca);p(Wt.$$.fragment,vf),vf.forEach(o),yf.forEach(o),pi=c(pr),Aa=s(pr,"SPAN",{});var _f=n(Aa);mi=l(_f,"Load a quantized model from the \u{1F917} Hub"),_f.forEach(o),pr.forEach(o),kn=c(e),S=s(e,"P",{});var No=n(S);bi=l(No,"You can load a quantized model from the Hub by using "),Ba=s(No,"CODE",{});var gf=n(Ba);yi=l(gf,"from_pretrained"),gf.forEach(o),vi=l(No," method. Make sure that the pushed weights are quantized, by checking that the attribute "),Ua=s(No,"CODE",{});var wf=n(Ua);_i=l(wf,"quantization_config"),wf.forEach(o),gi=l(No," is present in the model configuration object."),No.forEach(o),Jn=c(e),p(jt.$$.fragment,e),Cn=c(e),J=s(e,"P",{});var X=n(J);wi=l(X,"Note that in this case, you don\u2019t need to specify the arguments "),za=s(X,"CODE",{});var $f=n(za);$i=l($f,"load_in_8bit=True"),$f.forEach(o),Ei=l(X,", but you need to make sure that "),Na=s(X,"CODE",{});var Ef=n(Na);Mi=l(Ef,"bitsandbytes"),Ef.forEach(o),Ti=l(X," and "),qa=s(X,"CODE",{});var Mf=n(qa);ki=l(Mf,"accelerate"),Mf.forEach(o),Ji=l(X,` are installed.
Note also that `),Za=s(X,"CODE",{});var Tf=n(Za);Ci=l(Tf,"device_map"),Tf.forEach(o),Ai=l(X," is optional but setting "),Wa=s(X,"CODE",{});var kf=n(Wa);Bi=l(kf,"device_map = 'auto'"),kf.forEach(o),Ui=l(X," is prefered for inference as it will dispatch efficiently the model on the available ressources."),X.forEach(o),An=c(e),be=s(e,"H3",{class:!0});var mr=n(be);Ye=s(mr,"A",{id:!0,class:!0,href:!0});var Jf=n(Ye);ja=s(Jf,"SPAN",{});var Cf=n(ja);p(Pt.$$.fragment,Cf),Cf.forEach(o),Jf.forEach(o),zi=c(mr),Pa=s(mr,"SPAN",{});var Af=n(Pa);Ni=l(Af,"Advanced usecases"),Af.forEach(o),mr.forEach(o),Bn=c(e),wo=s(e,"P",{});var Bf=n(wo);qi=l(Bf,"This section is intended to advanced users, that want to explore what it is possible to do beyond loading and running 8-bit models."),Bf.forEach(o),Un=c(e),ye=s(e,"H4",{class:!0});var br=n(ye);He=s(br,"A",{id:!0,class:!0,href:!0});var Uf=n(He);Xa=s(Uf,"SPAN",{});var zf=n(Xa);p(Xt.$$.fragment,zf),zf.forEach(o),Uf.forEach(o),Zi=c(br),Le=s(br,"SPAN",{});var Vs=n(Le);Wi=l(Vs,"Offload between "),Fa=s(Vs,"CODE",{});var Nf=n(Fa);ji=l(Nf,"cpu"),Nf.forEach(o),Pi=l(Vs," and "),Va=s(Vs,"CODE",{});var qf=n(Va);Xi=l(qf,"gpu"),qf.forEach(o),Vs.forEach(o),br.forEach(o),zn=c(e),B=s(e,"P",{});var L=n(B);Fi=l(L,"One of the advanced usecase of this is being able to load a model and dispatch the weights between "),Ia=s(L,"CODE",{});var Zf=n(Ia);Vi=l(Zf,"CPU"),Zf.forEach(o),Ii=l(L," and "),Ra=s(L,"CODE",{});var Wf=n(Ra);Ri=l(Wf,"GPU"),Wf.forEach(o),xi=l(L,". Note that the weights that will be dispatched on CPU "),xa=s(L,"STRONG",{});var jf=n(xa);Di=l(jf,"will not"),jf.forEach(o),Qi=l(L," be converted in 8-bit, thus kept in "),Da=s(L,"CODE",{});var Pf=n(Da);Gi=l(Pf,"float32"),Pf.forEach(o),Si=l(L,". This feature is intended for users that want to fit a very large model and dispatch the model between GPU and CPU."),L.forEach(o),Nn=c(e),U=s(e,"P",{});var K=n(U);Oi=l(K,"First, load a "),Qa=s(K,"CODE",{});var Xf=n(Qa);Yi=l(Xf,"BitsAndBytesConfig"),Xf.forEach(o),Hi=l(K," from "),Ga=s(K,"CODE",{});var Ff=n(Ga);Li=l(Ff,"transformers"),Ff.forEach(o),Ki=l(K," and set the attribute "),Sa=s(K,"CODE",{});var Vf=n(Sa);ed=l(Vf,"llm_int8_enable_fp32_cpu_offload"),Vf.forEach(o),td=l(K," to "),Oa=s(K,"CODE",{});var If=n(Oa);od=l(If,"True"),If.forEach(o),ad=l(K,":"),K.forEach(o),qn=c(e),p(Ft.$$.fragment,e),Zn=c(e),O=s(e,"P",{});var qo=n(O);sd=l(qo,"Let\u2019s say you want to load "),Ya=s(qo,"CODE",{});var Rf=n(Ya);nd=l(Rf,"bigscience/bloom-1b7"),Rf.forEach(o),rd=l(qo," model, and you have just enough GPU RAM to fit the entire model except the "),Ha=s(qo,"CODE",{});var xf=n(Ha);ld=l(xf,"lm_head"),xf.forEach(o),id=l(qo,". Therefore write a custom device_map as follows:"),qo.forEach(o),Wn=c(e),p(Vt.$$.fragment,e),jn=c(e),$o=s(e,"P",{});var Df=n($o);dd=l(Df,"And load your model as follows:"),Df.forEach(o),Pn=c(e),p(It.$$.fragment,e),Xn=c(e),Eo=s(e,"P",{});var Qf=n(Eo);hd=l(Qf,"And that\u2019s it! Enjoy your model!"),Qf.forEach(o),Fn=c(e),ve=s(e,"H4",{class:!0});var yr=n(ve);Ke=s(yr,"A",{id:!0,class:!0,href:!0});var Gf=n(Ke);La=s(Gf,"SPAN",{});var Sf=n(La);p(Rt.$$.fragment,Sf),Sf.forEach(o),Gf.forEach(o),cd=c(yr),Mo=s(yr,"SPAN",{});var oc=n(Mo);fd=l(oc,"Play with "),Ka=s(oc,"CODE",{});var Of=n(Ka);ud=l(Of,"llm_int8_threshold"),Of.forEach(o),oc.forEach(o),yr.forEach(o),Vn=c(e),Y=s(e,"P",{});var Zo=n(Y);pd=l(Zo,"You can play with the "),es=s(Zo,"CODE",{});var Yf=n(es);md=l(Yf,"llm_int8_threshold"),Yf.forEach(o),bd=l(Zo,` argument to change the threshold of the outliers. An \u201Coutlier\u201D is a hidden state value that is greater than a certain threshold.
This corresponds to the outlier threshold for outlier detection as described in `),ts=s(Zo,"CODE",{});var Hf=n(ts);yd=l(Hf,"LLM.int8()"),Hf.forEach(o),vd=l(Zo,` paper. Any hidden states value that is above this threshold will be considered an outlier and the operation on those values will be done in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but there are some exceptional systematic outliers that are very differently distributed for large models. These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6, but a lower threshold might be needed for more unstable models (small models, fine-tuning).
This argument can impact the inference speed of the model. We suggest to play with this parameter to find which one is the best for your usecase.`),Zo.forEach(o),In=c(e),p(xt.$$.fragment,e),Rn=c(e),_e=s(e,"H4",{class:!0});var vr=n(_e);et=s(vr,"A",{id:!0,class:!0,href:!0});var Lf=n(et);os=s(Lf,"SPAN",{});var Kf=n(os);p(Dt.$$.fragment,Kf),Kf.forEach(o),Lf.forEach(o),_d=c(vr),as=s(vr,"SPAN",{});var eu=n(as);gd=l(eu,"Skip the conversion of some modules"),eu.forEach(o),vr.forEach(o),xn=c(e),ge=s(e,"P",{});var Is=n(ge);wd=l(Is,"Some models has several modules that needs to be not converted in 8-bit to ensure stability. For example Jukebox model has several "),ss=s(Is,"CODE",{});var tu=n(ss);$d=l(tu,"lm_head"),tu.forEach(o),Ed=l(Is," modules that should be skipped. Play with "),ns=s(Is,"CODE",{});var ou=n(ns);Md=l(ou,"llm_int8_skip_modules"),ou.forEach(o),Is.forEach(o),Dn=c(e),p(Qt.$$.fragment,e),Qn=c(e),we=s(e,"H4",{class:!0});var _r=n(we);tt=s(_r,"A",{id:!0,class:!0,href:!0});var au=n(tt);rs=s(au,"SPAN",{});var su=n(rs);p(Gt.$$.fragment,su),su.forEach(o),au.forEach(o),Td=c(_r),ls=s(_r,"SPAN",{});var nu=n(ls);kd=l(nu,"Fine-tune a model that has been loaded in 8-bit"),nu.forEach(o),_r.forEach(o),Gn=c(e),P=s(e,"P",{});var pt=n(P);Jd=l(pt,`With the official support of adapters in the Hugging Face ecosystem, you can fine-tune models that have been loaded in 8-bit.
This enables fine-tuning large models such as `),is=s(pt,"CODE",{});var ru=n(is);Cd=l(ru,"flan-t5-large"),ru.forEach(o),Ad=l(pt," or "),ds=s(pt,"CODE",{});var lu=n(ds);Bd=l(lu,"facebook/opt-6.7b"),lu.forEach(o),Ud=l(pt," in a single google Colab. Please have a look at "),St=s(pt,"A",{href:!0,rel:!0});var iu=n(St);hs=s(iu,"CODE",{});var du=n(hs);zd=l(du,"peft"),du.forEach(o),iu.forEach(o),Nd=l(pt," library for more details."),pt.forEach(o),Sn=c(e),C=s(e,"P",{});var F=n(C);qd=l(F,"Note that you don\u2019t need to pass "),cs=s(F,"CODE",{});var hu=n(cs);Zd=l(hu,"device_map"),hu.forEach(o),Wd=l(F," when loading the model for training. It will automatically load your model on your GPU. You can also set the device map to a specific device if needed (e.g. "),fs=s(F,"CODE",{});var cu=n(fs);jd=l(cu,"cuda:0"),cu.forEach(o),Pd=l(F,", "),us=s(F,"CODE",{});var fu=n(us);Xd=l(fu,"0"),fu.forEach(o),Fd=l(F,", "),ps=s(F,"CODE",{});var uu=n(ps);Vd=l(uu,"torch.device('cuda:0')"),uu.forEach(o),Id=l(F,"). Please note that "),ms=s(F,"CODE",{});var pu=n(ms);Rd=l(pu,"device_map=auto"),pu.forEach(o),xd=l(F," should be used for inference only."),F.forEach(o),On=c(e),$e=s(e,"H3",{class:!0});var gr=n($e);ot=s(gr,"A",{id:!0,class:!0,href:!0});var mu=n(ot);bs=s(mu,"SPAN",{});var bu=n(bs);p(Ot.$$.fragment,bu),bu.forEach(o),mu.forEach(o),Dd=c(gr),ys=s(gr,"SPAN",{});var yu=n(ys);Qd=l(yu,"BitsAndBytesConfig"),yu.forEach(o),gr.forEach(o),Yn=c(e),g=s(e,"DIV",{class:!0});var $=n(g);p(Yt.$$.fragment,$),Gd=c($),Ht=s($,"P",{});var wr=n(Ht);Sd=l(wr,`This is a wrapper class about all possible attributes and features that you can play with a model that has been
loaded using `),vs=s(wr,"CODE",{});var vu=n(vs);Od=l(vu,"bitsandbytes"),vu.forEach(o),Yd=l(wr,"."),wr.forEach(o),Hd=c($),Ee=s($,"P",{});var Wo=n(Ee);Ld=l(Wo,"This replaces "),_s=s(Wo,"CODE",{});var _u=n(_s);Kd=l(_u,"load_in_8bit"),_u.forEach(o),eh=l(Wo," or "),gs=s(Wo,"CODE",{});var gu=n(gs);th=l(gu,"load_in_4bit"),gu.forEach(o),oh=l(Wo,"therefore both options are mutually exclusive."),Wo.forEach(o),ah=c($),q=s($,"P",{});var ee=n(q);sh=l(ee,"Currently only supports "),ws=s(ee,"CODE",{});var wu=n(ws);nh=l(wu,"LLM.int8()"),wu.forEach(o),rh=l(ee,", "),$s=s(ee,"CODE",{});var $u=n($s);lh=l($u,"FP4"),$u.forEach(o),ih=l(ee,", and "),Es=s(ee,"CODE",{});var Eu=n(Es);dh=l(Eu,"NF4"),Eu.forEach(o),hh=l(ee," quantization. If more methods are added to "),Ms=s(ee,"CODE",{});var Mu=n(Ms);ch=l(Mu,"bitsandbytes"),Mu.forEach(o),fh=l(ee,`,
then more arguments will be added to this class.`),ee.forEach(o),uh=c($),at=s($,"DIV",{class:!0});var $r=n(at);p(Lt.$$.fragment,$r),ph=c($r),Kt=s($r,"P",{});var Er=n(Kt);mh=l(Er,"Instantiates a "),To=s(Er,"A",{href:!0});var Tu=n(To);bh=l(Tu,"BitsAndBytesConfig"),Tu.forEach(o),yh=l(Er," from a Python dictionary of parameters."),Er.forEach(o),$r.forEach(o),vh=c($),st=s($,"DIV",{class:!0});var Mr=n(st);p(eo.$$.fragment,Mr),_h=c(Mr),Me=s(Mr,"P",{});var jo=n(Me);gh=l(jo,"Returns "),Ts=s(jo,"CODE",{});var ku=n(Ts);wh=l(ku,"True"),ku.forEach(o),$h=l(jo," if the model is quantizable, "),ks=s(jo,"CODE",{});var Ju=n(ks);Eh=l(Ju,"False"),Ju.forEach(o),Mh=l(jo," otherwise."),jo.forEach(o),Mr.forEach(o),Th=c($),nt=s($,"DIV",{class:!0});var Tr=n(nt);p(to.$$.fragment,Tr),kh=c(Tr),Js=s(Tr,"P",{});var Cu=n(Js);Jh=l(Cu,"Safety checker that arguments are correct - also replaces some NoneType arguments with their default values."),Cu.forEach(o),Tr.forEach(o),Ch=c($),rt=s($,"DIV",{class:!0});var kr=n(rt);p(oo.$$.fragment,kr),Ah=c(kr),ao=s(kr,"P",{});var Jr=n(ao);Bh=l(Jr,`This method returns the quantization method used for the model. If the model is not quantizable, it returns
`),Cs=s(Jr,"CODE",{});var Au=n(Cs);Uh=l(Au,"None"),Au.forEach(o),zh=l(Jr,"."),Jr.forEach(o),kr.forEach(o),Nh=c($),lt=s($,"DIV",{class:!0});var Cr=n(lt);p(so.$$.fragment,Cr),qh=c(Cr),no=s(Cr,"P",{});var Ar=n(no);Zh=l(Ar,`Serializes this instance to a Python dictionary. Returns:
`),As=s(Ar,"CODE",{});var Bu=n(As);Wh=l(Bu,"Dict[str, Any]"),Bu.forEach(o),jh=l(Ar,": Dictionary of all the attributes that make up this configuration instance."),Ar.forEach(o),Cr.forEach(o),Ph=c($),it=s($,"DIV",{class:!0});var Br=n(it);p(ro.$$.fragment,Br),Xh=c(Br),Bs=s(Br,"P",{});var Uu=n(Bs);Fh=l(Uu,`Removes all attributes from config which correspond to the default config attributes for better readability and
serializes to a Python dictionary.`),Uu.forEach(o),Br.forEach(o),Vh=c($),dt=s($,"DIV",{class:!0});var Ur=n(dt);p(lo.$$.fragment,Ur),Ih=c(Ur),Us=s(Ur,"P",{});var zu=n(Us);Rh=l(zu,"Save this instance to a JSON file."),zu.forEach(o),Ur.forEach(o),xh=c($),ht=s($,"DIV",{class:!0});var zr=n(ht);p(io.$$.fragment,zr),Dh=c(zr),zs=s(zr,"P",{});var Nu=n(zs);Qh=l(Nu,"Serializes this instance to a JSON string."),Nu.forEach(o),zr.forEach(o),$.forEach(o),Hn=c(e),Te=s(e,"H2",{class:!0});var Nr=n(Te);ct=s(Nr,"A",{id:!0,class:!0,href:!0});var qu=n(ct);Ns=s(qu,"SPAN",{});var Zu=n(Ns);p(ho.$$.fragment,Zu),Zu.forEach(o),qu.forEach(o),Gh=c(Nr),ko=s(Nr,"SPAN",{});var ac=n(ko);Sh=l(ac,"Quantization with \u{1F917} "),qs=s(ac,"CODE",{});var Wu=n(qs);Oh=l(Wu,"optimum"),Wu.forEach(o),ac.forEach(o),Nr.forEach(o),Ln=c(e),H=s(e,"P",{});var Po=n(H);Yh=l(Po,"Please have a look at "),co=s(Po,"A",{href:!0,rel:!0});var ju=n(co);Hh=l(ju,"Optimum documentation"),ju.forEach(o),Lh=l(Po," to learn more about quantization methods that are supported by "),Zs=s(Po,"CODE",{});var Pu=n(Zs);Kh=l(Pu,"optimum"),Pu.forEach(o),ec=l(Po," and see if these are applicable for your usecase."),Po.forEach(o),this.h()},h(){f(_,"name","hf:doc:metadata"),f(_,"content",JSON.stringify(Su)),f(E,"id","quantize-transformers-models"),f(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(E,"href","#quantize-transformers-models"),f(w,"class","relative group"),f(A,"id","bitsandbytes-integration"),f(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(A,"href","#bitsandbytes-integration"),f(N,"class","relative group"),f(yt,"href","https://arxiv.org/abs/2208.07339"),f(yt,"rel","nofollow"),f(vt,"href","https://huggingface.co/blog/hf-bitsandbytes-integration"),f(vt,"rel","nofollow"),f(_t,"href","https://huggingface.co/docs/accelerate/main/en/usage_guides/quantization"),f(_t,"rel","nofollow"),f(ze,"id","fp4-quantization"),f(ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ze,"href","#fp4-quantization"),f(re,"class","relative group"),f(Ne,"id","requirements"),f(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ne,"href","#requirements"),f(le,"class","relative group"),f(je,"id","load-a-large-model-in-4bit"),f(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(je,"href","#load-a-large-model-in-4bit"),f(ie,"class","relative group"),f(Xe,"id","load-a-large-model-in-8bit"),f(Xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Xe,"href","#load-a-large-model-in-8bit"),f(de,"class","relative group"),f(bo,"href","/docs/transformers/v4.31.0/en/main_classes/model#transformers.PreTrainedModel"),f(Re,"id","advanced-usecases"),f(Re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Re,"href","#advanced-usecases"),f(he,"class","relative group"),f(xe,"id","change-the-compute-dtype"),f(xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(xe,"href","#change-the-compute-dtype"),f(ce,"class","relative group"),f(De,"id","using-nf4-normal-float-4-data-type"),f(De,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(De,"href","#using-nf4-normal-float-4-data-type"),f(fe,"class","relative group"),f(Qe,"id","use-nested-quantization-for-more-memory-efficient-inference"),f(Qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Qe,"href","#use-nested-quantization-for-more-memory-efficient-inference"),f(ue,"class","relative group"),f(Ge,"id","push-quantized-models-on-the-hub"),f(Ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ge,"href","#push-quantized-models-on-the-hub"),f(pe,"class","relative group"),f(Oe,"id","load-a-quantized-model-from-the-hub"),f(Oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Oe,"href","#load-a-quantized-model-from-the-hub"),f(me,"class","relative group"),f(Ye,"id","advanced-usecases"),f(Ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ye,"href","#advanced-usecases"),f(be,"class","relative group"),f(He,"id","offload-between-cpu-and-gpu"),f(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(He,"href","#offload-between-cpu-and-gpu"),f(ye,"class","relative group"),f(Ke,"id","play-with-llmint8threshold"),f(Ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ke,"href","#play-with-llmint8threshold"),f(ve,"class","relative group"),f(et,"id","skip-the-conversion-of-some-modules"),f(et,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(et,"href","#skip-the-conversion-of-some-modules"),f(_e,"class","relative group"),f(tt,"id","finetune-a-model-that-has-been-loaded-in-8bit"),f(tt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(tt,"href","#finetune-a-model-that-has-been-loaded-in-8bit"),f(we,"class","relative group"),f(St,"href","https://github.com/huggingface/peft"),f(St,"rel","nofollow"),f(ot,"id","transformers.BitsAndBytesConfig"),f(ot,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ot,"href","#transformers.BitsAndBytesConfig"),f($e,"class","relative group"),f(To,"href","/docs/transformers/v4.31.0/en/main_classes/quantization#transformers.BitsAndBytesConfig"),f(at,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(st,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(nt,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(rt,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(lt,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(it,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(dt,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(ht,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(g,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(ct,"id","quantization-with-optimum"),f(ct,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ct,"href","#quantization-with-optimum"),f(Te,"class","relative group"),f(co,"href","https://huggingface.co/docs/optimum/index"),f(co,"rel","nofollow")},m(e,i){t(document.head,_),d(e,k,i),d(e,w,i),t(w,E),t(E,te),m(z,te,null),t(w,mt),t(w,oe),t(oe,ae),d(e,Ce,i),d(e,N,i),t(N,A),t(A,se),m(V,se,null),t(N,ne),t(N,M),t(M,Ae),t(Ae,uo),t(M,po),d(e,bt,i),d(e,W,i),t(W,qr),t(W,Xo),t(Xo,Zr),t(W,Wr),t(W,Fo),t(Fo,jr),t(W,Pr),t(W,Vo),t(Vo,Xr),t(W,Fr),d(e,Rs,i),d(e,I,i),t(I,Vr),t(I,yt),t(yt,Ir),t(I,Rr),t(I,vt),t(vt,xr),t(I,Dr),d(e,xs,i),d(e,R,i),t(R,Qr),t(R,Io),t(Io,Gr),t(R,Sr),t(R,Ro),t(Ro,Or),t(R,Yr),d(e,Ds,i),d(e,Be,i),t(Be,Hr),t(Be,_t),t(_t,Lr),t(Be,Kr),d(e,Qs,i),d(e,Ue,i),t(Ue,el),t(Ue,xo),t(xo,tl),t(Ue,ol),d(e,Gs,i),d(e,re,i),t(re,ze),t(ze,Do),m(gt,Do,null),t(re,al),t(re,Qo),t(Qo,sl),d(e,Ss,i),d(e,le,i),t(le,Ne),t(Ne,Go),m(wt,Go,null),t(le,nl),t(le,So),t(So,rl),d(e,Os,i),d(e,mo,i),t(mo,ll),d(e,Ys,i),d(e,x,i),t(x,Oo),t(Oo,qe),t(qe,il),t(qe,Yo),t(Yo,dl),t(qe,hl),t(qe,Ho),t(Ho,cl),t(x,fl),t(x,Lo),t(Lo,Ze),t(Ze,ul),t(Ze,Ko),t(Ko,pl),t(Ze,ml),t(Ze,ea),t(ea,bl),t(x,yl),t(x,ta),t(ta,We),t(We,vl),t(We,oa),t(oa,_l),t(We,gl),t(We,aa),t(aa,wl),d(e,Hs,i),d(e,ie,i),t(ie,je),t(je,sa),m($t,sa,null),t(ie,$l),t(ie,na),t(na,El),d(e,Ls,i),d(e,D,i),t(D,Ml),t(D,ra),t(ra,Tl),t(D,kl),t(D,la),t(la,Jl),t(D,Cl),d(e,Ks,i),m(Et,e,i),d(e,en,i),m(Pe,e,i),d(e,tn,i),d(e,de,i),t(de,Xe),t(Xe,ia),m(Mt,ia,null),t(de,Al),t(de,da),t(da,Bl),d(e,on,i),d(e,Q,i),t(Q,Ul),t(Q,ha),t(ha,zl),t(Q,Nl),t(Q,ca),t(ca,ql),t(Q,Zl),d(e,an,i),m(Tt,e,i),d(e,sn,i),d(e,Fe,i),t(Fe,Wl),t(Fe,bo),t(bo,jl),t(Fe,Pl),d(e,nn,i),d(e,Ve,i),t(Ve,Xl),t(Ve,fa),t(fa,Fl),t(Ve,Vl),d(e,rn,i),m(kt,e,i),d(e,ln,i),d(e,yo,i),t(yo,Il),d(e,dn,i),m(Ie,e,i),d(e,hn,i),d(e,he,i),t(he,Re),t(Re,ua),m(Jt,ua,null),t(he,Rl),t(he,pa),t(pa,xl),d(e,cn,i),d(e,vo,i),t(vo,Dl),d(e,fn,i),d(e,ce,i),t(ce,xe),t(xe,ma),m(Ct,ma,null),t(ce,Ql),t(ce,ba),t(ba,Gl),d(e,un,i),d(e,G,i),t(G,Sl),t(G,ya),t(ya,Ol),t(G,Yl),t(G,va),t(va,Hl),t(G,Ll),d(e,pn,i),m(At,e,i),d(e,mn,i),d(e,fe,i),t(fe,De),t(De,_a),m(Bt,_a,null),t(fe,Kl),t(fe,ga),t(ga,ei),d(e,bn,i),d(e,_o,i),t(_o,ti),d(e,yn,i),m(Ut,e,i),d(e,vn,i),d(e,ue,i),t(ue,Qe),t(Qe,wa),m(zt,wa,null),t(ue,oi),t(ue,$a),t($a,ai),d(e,_n,i),d(e,go,i),t(go,si),d(e,gn,i),m(Nt,e,i),d(e,wn,i),d(e,pe,i),t(pe,Ge),t(Ge,Ea),m(qt,Ea,null),t(pe,ni),t(pe,Ma),t(Ma,ri),d(e,$n,i),d(e,j,i),t(j,li),t(j,Ta),t(Ta,ii),t(j,di),t(j,ka),t(ka,hi),t(j,ci),t(j,Ja),t(Ja,fi),t(j,ui),d(e,En,i),m(Zt,e,i),d(e,Mn,i),m(Se,e,i),d(e,Tn,i),d(e,me,i),t(me,Oe),t(Oe,Ca),m(Wt,Ca,null),t(me,pi),t(me,Aa),t(Aa,mi),d(e,kn,i),d(e,S,i),t(S,bi),t(S,Ba),t(Ba,yi),t(S,vi),t(S,Ua),t(Ua,_i),t(S,gi),d(e,Jn,i),m(jt,e,i),d(e,Cn,i),d(e,J,i),t(J,wi),t(J,za),t(za,$i),t(J,Ei),t(J,Na),t(Na,Mi),t(J,Ti),t(J,qa),t(qa,ki),t(J,Ji),t(J,Za),t(Za,Ci),t(J,Ai),t(J,Wa),t(Wa,Bi),t(J,Ui),d(e,An,i),d(e,be,i),t(be,Ye),t(Ye,ja),m(Pt,ja,null),t(be,zi),t(be,Pa),t(Pa,Ni),d(e,Bn,i),d(e,wo,i),t(wo,qi),d(e,Un,i),d(e,ye,i),t(ye,He),t(He,Xa),m(Xt,Xa,null),t(ye,Zi),t(ye,Le),t(Le,Wi),t(Le,Fa),t(Fa,ji),t(Le,Pi),t(Le,Va),t(Va,Xi),d(e,zn,i),d(e,B,i),t(B,Fi),t(B,Ia),t(Ia,Vi),t(B,Ii),t(B,Ra),t(Ra,Ri),t(B,xi),t(B,xa),t(xa,Di),t(B,Qi),t(B,Da),t(Da,Gi),t(B,Si),d(e,Nn,i),d(e,U,i),t(U,Oi),t(U,Qa),t(Qa,Yi),t(U,Hi),t(U,Ga),t(Ga,Li),t(U,Ki),t(U,Sa),t(Sa,ed),t(U,td),t(U,Oa),t(Oa,od),t(U,ad),d(e,qn,i),m(Ft,e,i),d(e,Zn,i),d(e,O,i),t(O,sd),t(O,Ya),t(Ya,nd),t(O,rd),t(O,Ha),t(Ha,ld),t(O,id),d(e,Wn,i),m(Vt,e,i),d(e,jn,i),d(e,$o,i),t($o,dd),d(e,Pn,i),m(It,e,i),d(e,Xn,i),d(e,Eo,i),t(Eo,hd),d(e,Fn,i),d(e,ve,i),t(ve,Ke),t(Ke,La),m(Rt,La,null),t(ve,cd),t(ve,Mo),t(Mo,fd),t(Mo,Ka),t(Ka,ud),d(e,Vn,i),d(e,Y,i),t(Y,pd),t(Y,es),t(es,md),t(Y,bd),t(Y,ts),t(ts,yd),t(Y,vd),d(e,In,i),m(xt,e,i),d(e,Rn,i),d(e,_e,i),t(_e,et),t(et,os),m(Dt,os,null),t(_e,_d),t(_e,as),t(as,gd),d(e,xn,i),d(e,ge,i),t(ge,wd),t(ge,ss),t(ss,$d),t(ge,Ed),t(ge,ns),t(ns,Md),d(e,Dn,i),m(Qt,e,i),d(e,Qn,i),d(e,we,i),t(we,tt),t(tt,rs),m(Gt,rs,null),t(we,Td),t(we,ls),t(ls,kd),d(e,Gn,i),d(e,P,i),t(P,Jd),t(P,is),t(is,Cd),t(P,Ad),t(P,ds),t(ds,Bd),t(P,Ud),t(P,St),t(St,hs),t(hs,zd),t(P,Nd),d(e,Sn,i),d(e,C,i),t(C,qd),t(C,cs),t(cs,Zd),t(C,Wd),t(C,fs),t(fs,jd),t(C,Pd),t(C,us),t(us,Xd),t(C,Fd),t(C,ps),t(ps,Vd),t(C,Id),t(C,ms),t(ms,Rd),t(C,xd),d(e,On,i),d(e,$e,i),t($e,ot),t(ot,bs),m(Ot,bs,null),t($e,Dd),t($e,ys),t(ys,Qd),d(e,Yn,i),d(e,g,i),m(Yt,g,null),t(g,Gd),t(g,Ht),t(Ht,Sd),t(Ht,vs),t(vs,Od),t(Ht,Yd),t(g,Hd),t(g,Ee),t(Ee,Ld),t(Ee,_s),t(_s,Kd),t(Ee,eh),t(Ee,gs),t(gs,th),t(Ee,oh),t(g,ah),t(g,q),t(q,sh),t(q,ws),t(ws,nh),t(q,rh),t(q,$s),t($s,lh),t(q,ih),t(q,Es),t(Es,dh),t(q,hh),t(q,Ms),t(Ms,ch),t(q,fh),t(g,uh),t(g,at),m(Lt,at,null),t(at,ph),t(at,Kt),t(Kt,mh),t(Kt,To),t(To,bh),t(Kt,yh),t(g,vh),t(g,st),m(eo,st,null),t(st,_h),t(st,Me),t(Me,gh),t(Me,Ts),t(Ts,wh),t(Me,$h),t(Me,ks),t(ks,Eh),t(Me,Mh),t(g,Th),t(g,nt),m(to,nt,null),t(nt,kh),t(nt,Js),t(Js,Jh),t(g,Ch),t(g,rt),m(oo,rt,null),t(rt,Ah),t(rt,ao),t(ao,Bh),t(ao,Cs),t(Cs,Uh),t(ao,zh),t(g,Nh),t(g,lt),m(so,lt,null),t(lt,qh),t(lt,no),t(no,Zh),t(no,As),t(As,Wh),t(no,jh),t(g,Ph),t(g,it),m(ro,it,null),t(it,Xh),t(it,Bs),t(Bs,Fh),t(g,Vh),t(g,dt),m(lo,dt,null),t(dt,Ih),t(dt,Us),t(Us,Rh),t(g,xh),t(g,ht),m(io,ht,null),t(ht,Dh),t(ht,zs),t(zs,Qh),d(e,Hn,i),d(e,Te,i),t(Te,ct),t(ct,Ns),m(ho,Ns,null),t(Te,Gh),t(Te,ko),t(ko,Sh),t(ko,qs),t(qs,Oh),d(e,Ln,i),d(e,H,i),t(H,Yh),t(H,co),t(co,Hh),t(H,Lh),t(H,Zs),t(Zs,Kh),t(H,ec),Kn=!0},p(e,[i]){const fo={};i&2&&(fo.$$scope={dirty:i,ctx:e}),Pe.$set(fo);const Ws={};i&2&&(Ws.$$scope={dirty:i,ctx:e}),Ie.$set(Ws);const js={};i&2&&(js.$$scope={dirty:i,ctx:e}),Se.$set(js)},i(e){Kn||(b(z.$$.fragment,e),b(V.$$.fragment,e),b(gt.$$.fragment,e),b(wt.$$.fragment,e),b($t.$$.fragment,e),b(Et.$$.fragment,e),b(Pe.$$.fragment,e),b(Mt.$$.fragment,e),b(Tt.$$.fragment,e),b(kt.$$.fragment,e),b(Ie.$$.fragment,e),b(Jt.$$.fragment,e),b(Ct.$$.fragment,e),b(At.$$.fragment,e),b(Bt.$$.fragment,e),b(Ut.$$.fragment,e),b(zt.$$.fragment,e),b(Nt.$$.fragment,e),b(qt.$$.fragment,e),b(Zt.$$.fragment,e),b(Se.$$.fragment,e),b(Wt.$$.fragment,e),b(jt.$$.fragment,e),b(Pt.$$.fragment,e),b(Xt.$$.fragment,e),b(Ft.$$.fragment,e),b(Vt.$$.fragment,e),b(It.$$.fragment,e),b(Rt.$$.fragment,e),b(xt.$$.fragment,e),b(Dt.$$.fragment,e),b(Qt.$$.fragment,e),b(Gt.$$.fragment,e),b(Ot.$$.fragment,e),b(Yt.$$.fragment,e),b(Lt.$$.fragment,e),b(eo.$$.fragment,e),b(to.$$.fragment,e),b(oo.$$.fragment,e),b(so.$$.fragment,e),b(ro.$$.fragment,e),b(lo.$$.fragment,e),b(io.$$.fragment,e),b(ho.$$.fragment,e),Kn=!0)},o(e){y(z.$$.fragment,e),y(V.$$.fragment,e),y(gt.$$.fragment,e),y(wt.$$.fragment,e),y($t.$$.fragment,e),y(Et.$$.fragment,e),y(Pe.$$.fragment,e),y(Mt.$$.fragment,e),y(Tt.$$.fragment,e),y(kt.$$.fragment,e),y(Ie.$$.fragment,e),y(Jt.$$.fragment,e),y(Ct.$$.fragment,e),y(At.$$.fragment,e),y(Bt.$$.fragment,e),y(Ut.$$.fragment,e),y(zt.$$.fragment,e),y(Nt.$$.fragment,e),y(qt.$$.fragment,e),y(Zt.$$.fragment,e),y(Se.$$.fragment,e),y(Wt.$$.fragment,e),y(jt.$$.fragment,e),y(Pt.$$.fragment,e),y(Xt.$$.fragment,e),y(Ft.$$.fragment,e),y(Vt.$$.fragment,e),y(It.$$.fragment,e),y(Rt.$$.fragment,e),y(xt.$$.fragment,e),y(Dt.$$.fragment,e),y(Qt.$$.fragment,e),y(Gt.$$.fragment,e),y(Ot.$$.fragment,e),y(Yt.$$.fragment,e),y(Lt.$$.fragment,e),y(eo.$$.fragment,e),y(to.$$.fragment,e),y(oo.$$.fragment,e),y(so.$$.fragment,e),y(ro.$$.fragment,e),y(lo.$$.fragment,e),y(io.$$.fragment,e),y(ho.$$.fragment,e),Kn=!1},d(e){o(_),e&&o(k),e&&o(w),v(z),e&&o(Ce),e&&o(N),v(V),e&&o(bt),e&&o(W),e&&o(Rs),e&&o(I),e&&o(xs),e&&o(R),e&&o(Ds),e&&o(Be),e&&o(Qs),e&&o(Ue),e&&o(Gs),e&&o(re),v(gt),e&&o(Ss),e&&o(le),v(wt),e&&o(Os),e&&o(mo),e&&o(Ys),e&&o(x),e&&o(Hs),e&&o(ie),v($t),e&&o(Ls),e&&o(D),e&&o(Ks),v(Et,e),e&&o(en),v(Pe,e),e&&o(tn),e&&o(de),v(Mt),e&&o(on),e&&o(Q),e&&o(an),v(Tt,e),e&&o(sn),e&&o(Fe),e&&o(nn),e&&o(Ve),e&&o(rn),v(kt,e),e&&o(ln),e&&o(yo),e&&o(dn),v(Ie,e),e&&o(hn),e&&o(he),v(Jt),e&&o(cn),e&&o(vo),e&&o(fn),e&&o(ce),v(Ct),e&&o(un),e&&o(G),e&&o(pn),v(At,e),e&&o(mn),e&&o(fe),v(Bt),e&&o(bn),e&&o(_o),e&&o(yn),v(Ut,e),e&&o(vn),e&&o(ue),v(zt),e&&o(_n),e&&o(go),e&&o(gn),v(Nt,e),e&&o(wn),e&&o(pe),v(qt),e&&o($n),e&&o(j),e&&o(En),v(Zt,e),e&&o(Mn),v(Se,e),e&&o(Tn),e&&o(me),v(Wt),e&&o(kn),e&&o(S),e&&o(Jn),v(jt,e),e&&o(Cn),e&&o(J),e&&o(An),e&&o(be),v(Pt),e&&o(Bn),e&&o(wo),e&&o(Un),e&&o(ye),v(Xt),e&&o(zn),e&&o(B),e&&o(Nn),e&&o(U),e&&o(qn),v(Ft,e),e&&o(Zn),e&&o(O),e&&o(Wn),v(Vt,e),e&&o(jn),e&&o($o),e&&o(Pn),v(It,e),e&&o(Xn),e&&o(Eo),e&&o(Fn),e&&o(ve),v(Rt),e&&o(Vn),e&&o(Y),e&&o(In),v(xt,e),e&&o(Rn),e&&o(_e),v(Dt),e&&o(xn),e&&o(ge),e&&o(Dn),v(Qt,e),e&&o(Qn),e&&o(we),v(Gt),e&&o(Gn),e&&o(P),e&&o(Sn),e&&o(C),e&&o(On),e&&o($e),v(Ot),e&&o(Yn),e&&o(g),v(Yt),v(Lt),v(eo),v(to),v(oo),v(so),v(ro),v(lo),v(io),e&&o(Hn),e&&o(Te),v(ho),e&&o(Ln),e&&o(H)}}}const Su={local:"quantize-transformers-models",sections:[{local:"bitsandbytes-integration",sections:[{local:"fp4-quantization",sections:[{local:"requirements",title:"Requirements"},{local:"load-a-large-model-in-4bit",title:"Load a large model in 4bit"}],title:"FP4 quantization "},{local:"load-a-large-model-in-8bit",sections:[{local:"advanced-usecases",sections:[{local:"change-the-compute-dtype",title:"Change the compute dtype"},{local:"using-nf4-normal-float-4-data-type",title:"Using NF4 (Normal Float 4) data type "},{local:"use-nested-quantization-for-more-memory-efficient-inference",title:"Use nested quantization for more memory efficient inference"}],title:"Advanced usecases"}],title:"Load a large model in 8bit"},{local:"push-quantized-models-on-the-hub",title:"Push quantized models on the \u{1F917} Hub"},{local:"load-a-quantized-model-from-the-hub",title:"Load a quantized model from the \u{1F917} Hub"},{local:"advanced-usecases",sections:[{local:"offload-between-cpu-and-gpu",title:"Offload between `cpu` and `gpu`"},{local:"play-with-llmint8threshold",title:"Play with `llm_int8_threshold`"},{local:"skip-the-conversion-of-some-modules",title:"Skip the conversion of some modules"},{local:"finetune-a-model-that-has-been-loaded-in-8bit",title:"Fine-tune a model that has been loaded in 8-bit"}],title:"Advanced usecases"},{local:"transformers.BitsAndBytesConfig",title:"BitsAndBytesConfig"}],title:"`bitsandbytes` Integration"},{local:"quantization-with-optimum",title:"Quantization with \u{1F917} `optimum` "}],title:"Quantize \u{1F917} Transformers models"};function Ou(Je){return Ru(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class tp extends Xu{constructor(_){super();Fu(this,_,Ou,Gu,Vu,{})}}export{tp as default,Su as metadata};
