import{S as Dl,i as xl,s as Ll,e as r,k as h,w as m,t as a,M as jl,c as n,d as o,m as f,a as i,x as c,h as s,b as u,G as t,g as d,y as p,q as _,o as b,B as g,v as Ol}from"../../chunks/vendor-hf-doc-builder.js";import{T as Ml}from"../../chunks/Tip-hf-doc-builder.js";import{D as Fo}from"../../chunks/Docstring-hf-doc-builder.js";import{C as ee}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as T}from"../../chunks/IconCopyLink-hf-doc-builder.js";function Sl(it){let v,z;return{c(){v=r("p"),z=a("Note that once a model has been loaded in 8-bit it is currently not possible to push the quantized weights on the Hub. Note also that you cannot train 8-bit weights as this is not supported yet. However you can use 8-bit models to train extra parameters, this will be covered in the next section.")},l(y){v=n(y,"P",{});var $=i(v);z=s($,"Note that once a model has been loaded in 8-bit it is currently not possible to push the quantized weights on the Hub. Note also that you cannot train 8-bit weights as this is not supported yet. However you can use 8-bit models to train extra parameters, this will be covered in the next section."),$.forEach(o)},m(y,$){d(y,v,$),t(v,z)},d(y){y&&o(v)}}}function Nl(it){let v,z;return{c(){v=r("p"),z=a("Pushing 8bit models on the Hub is strongely encouraged for large models. This will allow the community to benefit from the memory footprint reduction and loading for example large models on a Google Colab.")},l(y){v=n(y,"P",{});var $=i(v);z=s($,"Pushing 8bit models on the Hub is strongely encouraged for large models. This will allow the community to benefit from the memory footprint reduction and loading for example large models on a Google Colab."),$.forEach(o)},m(y,$){d(y,v,$),t(v,z)},d(y){y&&o(v)}}}function Fl(it){let v,z,y,$,Et,Ce,Za,At,es,Ho,U,te,kt,ze,ts,lt,Ct,os,as,Uo,q,ss,zt,rs,ns,qt,is,ls,Pt,ds,hs,Io,D,fs,qe,us,ms,Pe,cs,ps,Go,oe,_s,Bt,bs,gs,Yo,I,ae,Tt,Be,vs,Mt,ys,Jo,x,ws,Dt,$s,Es,xt,As,ks,Vo,Te,Qo,se,Cs,dt,zs,qs,Wo,re,Ps,Lt,Bs,Ts,Ro,Me,Ko,ht,Ms,Xo,ne,Zo,G,ie,jt,De,Ds,Ot,xs,ea,P,Ls,St,js,Os,Nt,Ss,Ns,Ft,Fs,Hs,ta,xe,oa,le,aa,Y,de,Ht,Le,Us,Ut,Is,sa,L,Gs,It,Ys,Js,Gt,Vs,Qs,ra,je,na,A,Ws,Yt,Rs,Ks,Jt,Xs,Zs,Vt,er,tr,Qt,or,ar,ia,J,he,Wt,Oe,sr,Rt,rr,la,ft,nr,da,V,fe,Kt,Se,ir,ue,lr,Xt,dr,hr,Zt,fr,ha,k,ur,eo,mr,cr,to,pr,_r,oo,br,gr,ao,vr,yr,fa,C,wr,so,$r,Er,ro,Ar,kr,no,Cr,zr,io,qr,Pr,ua,Ne,ma,j,Br,lo,Tr,Mr,ho,Dr,xr,ca,Fe,pa,ut,Lr,_a,He,ba,mt,jr,ga,Q,me,fo,Ue,Or,ct,Sr,uo,Nr,va,O,Fr,mo,Hr,Ur,co,Ir,Gr,ya,Ie,wa,W,ce,po,Ge,Yr,_o,Jr,$a,R,Vr,bo,Qr,Wr,go,Rr,Ea,Ye,Aa,K,pe,vo,Je,Kr,yo,Xr,ka,B,Zr,wo,en,tn,$o,on,an,Ve,Eo,sn,rn,Ca,X,_e,Ao,Qe,nn,ko,ln,za,w,We,dn,Re,hn,Co,fn,un,mn,Ke,cn,zo,pn,_n,bn,M,gn,qo,vn,yn,Po,wn,$n,Bo,En,An,kn,be,Xe,Cn,Ze,zn,pt,qn,Pn,Bn,ge,et,Tn,To,Mn,Dn,ve,tt,xn,ot,Ln,Mo,jn,On,Sn,ye,at,Nn,Do,Fn,qa,Z,we,xo,st,Hn,_t,Un,Lo,In,Pa,S,Gn,rt,Yn,Jn,jo,Vn,Qn,Ba;return Ce=new T({}),ze=new T({}),Be=new T({}),Te=new ee({props:{code:`# pip install transformers accelerate bitsandbytes
from transformers import AutoModelForCausalLM, AutoTokenizer

model_id = "bigscience/bloom-1b7"

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_8bit=True)`,highlighted:`<span class="hljs-comment"># pip install transformers accelerate bitsandbytes</span>
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),Me=new ee({props:{code:"print(model.get_memory_footprint())",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())'}}),ne=new Ml({props:{warning:!0,$$slots:{default:[Sl]},$$scope:{ctx:it}}}),De=new T({}),xe=new ee({props:{code:`from transformers import AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained("bigscience/bloom-560m", device_map="auto", load_in_8bit=True)
tokenizer = AutoTokenizer.from_pretrained("bigscience/bloom-560m")

model.push_to_hub("bloom-560m-8bit")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>)

model.push_to_hub(<span class="hljs-string">&quot;bloom-560m-8bit&quot;</span>)`}}),le=new Ml({props:{warning:!0,$$slots:{default:[Nl]},$$scope:{ctx:it}}}),Le=new T({}),je=new ee({props:{code:`from transformers import AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained("{your_username}/bloom-560m-8bit")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/bloom-560m-8bit&quot;</span>)`}}),Oe=new T({}),Se=new T({}),Ne=new ee({props:{code:`from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=<span class="hljs-literal">True</span>)`}}),Fe=new ee({props:{code:`device_map = {
    "transformer.word_embeddings": 0,
    "transformer.word_embeddings_layernorm": 0,
    "lm_head": "cpu",
    "transformer.h": 0,
    "transformer.ln_f": 0,
}`,highlighted:`device_map = {
    <span class="hljs-string">&quot;transformer.word_embeddings&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.word_embeddings_layernorm&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;lm_head&quot;</span>: <span class="hljs-string">&quot;cpu&quot;</span>,
    <span class="hljs-string">&quot;transformer.h&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.ln_f&quot;</span>: <span class="hljs-number">0</span>,
}`}}),He=new ee({props:{code:`model_8bit = AutoModelForCausalLM.from_pretrained(
    "bigscience/bloom-1b7",
    device_map=device_map,
    quantization_config=quantization_config,
)`,highlighted:`model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>,
    device_map=device_map,
    quantization_config=quantization_config,
)`}}),Ue=new T({}),Ie=new ee({props:{code:`from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = "bigscience/bloom-1b7"

quantization_config = BitsAndBytesConfig(
    llm_int8_threshold=10,
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)
tokenizer = AutoTokenizer.from_pretrained(model_id)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)
tokenizer = AutoTokenizer.from_pretrained(model_id)`}}),Ge=new T({}),Ye=new ee({props:{code:`from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = "bigscience/bloom-1b7"

quantization_config = BitsAndBytesConfig(
    llm_int8_skip_modules=["lm_head"],
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)
tokenizer = AutoTokenizer.from_pretrained(model_id)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_skip_modules=[<span class="hljs-string">&quot;lm_head&quot;</span>],
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)
tokenizer = AutoTokenizer.from_pretrained(model_id)`}}),Je=new T({}),Qe=new T({}),We=new Fo({props:{name:"class transformers.BitsAndBytesConfig",anchor:"transformers.BitsAndBytesConfig",parameters:[{name:"load_in_8bit",val:" = False"},{name:"llm_int8_threshold",val:" = 6.0"},{name:"llm_int8_skip_modules",val:" = None"},{name:"llm_int8_enable_fp32_cpu_offload",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BitsAndBytesConfig.load_in_8bit",description:`<strong>load_in_8bit</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag is used to enable 8-bit quantization with LLM.int8().`,name:"load_in_8bit"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_threshold",description:`<strong>llm_int8_threshold</strong> (<code>float</code>, <em>optional</em>, defaults to 6) &#x2014;
This corresponds to the outlier threshold for outlier detection as described in <code>LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale</code> paper: <a href="https://arxiv.org/abs/2208.07339" rel="nofollow">https://arxiv.org/abs/2208.07339</a> Any hidden states value
that is above this threshold will be considered an outlier and the operation on those values will be done
in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but
there are some exceptional systematic outliers that are very differently distributed for large models.
These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of
magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6,
but a lower threshold might be needed for more unstable models (small models, fine-tuning).`,name:"llm_int8_threshold"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_skip_modules",description:`<strong>llm_int8_skip_modules</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
An explicit list of the modules that we do not want to convert in 8-bit. This is useful for models such as
Jukebox that has several heads in different places and not necessarily at the last position. For example
for <code>CausalLM</code> models, the last <code>lm_head</code> is kept in its original <code>dtype</code>.`,name:"llm_int8_skip_modules"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_enable_fp32_cpu_offload",description:`<strong>llm_int8_enable_fp32_cpu_offload</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag is used for advanced use cases and users that are aware of this feature. If you want to split
your model in different parts and run some parts in int8 on GPU and some parts in fp32 on CPU, you can use
this flag. This is useful for offloading large models such as <code>google/flan-t5-xxl</code>. Note that the int8
operations will not be run on CPU.`,name:"llm_int8_enable_fp32_cpu_offload"},{anchor:"transformers.BitsAndBytesConfig.kwargs",description:`<strong>kwargs</strong> (<code>Dict[str, Any]</code>, <em>optional</em>) &#x2014;
Additional parameters from which to initialize the configuration object.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.29.0/src/transformers/utils/quantization_config.py#L30"}}),Xe=new Fo({props:{name:"from_dict",anchor:"transformers.BitsAndBytesConfig.from_dict",parameters:[{name:"config_dict",val:""},{name:"return_unused_kwargs",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BitsAndBytesConfig.from_dict.config_dict",description:`<strong>config_dict</strong> (<code>Dict[str, Any]</code>) &#x2014;
Dictionary that will be used to instantiate the configuration object.`,name:"config_dict"},{anchor:"transformers.BitsAndBytesConfig.from_dict.return_unused_kwargs",description:`<strong>return_unused_kwargs</strong> (<code>bool</code>) &#x2014;
Whether or not to return a list of unused keyword arguments. Used for <code>from_pretrained</code> method in
<code>PreTrainedModel</code>.`,name:"return_unused_kwargs"},{anchor:"transformers.BitsAndBytesConfig.from_dict.kwargs",description:`<strong>kwargs</strong> (<code>Dict[str, Any]</code>) &#x2014;
Additional parameters from which to initialize the configuration object.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.29.0/src/transformers/utils/quantization_config.py#L93",returnDescription:`
<p>The configuration object instantiated from those parameters.</p>
`,returnType:`
<p><a
  href="/docs/transformers/v4.29.0/en/main_classes/quantization#transformers.BitsAndBytesConfig"
>BitsAndBytesConfig</a></p>
`}}),et=new Fo({props:{name:"post_init",anchor:"transformers.BitsAndBytesConfig.post_init",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.29.0/src/transformers/utils/quantization_config.py#L80"}}),tt=new Fo({props:{name:"to_dict",anchor:"transformers.BitsAndBytesConfig.to_dict",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.29.0/src/transformers/utils/quantization_config.py#L142"}}),at=new Fo({props:{name:"to_json_file",anchor:"transformers.BitsAndBytesConfig.to_json_file",parameters:[{name:"json_file_path",val:": typing.Union[str, os.PathLike]"}],parametersDescription:[{anchor:"transformers.BitsAndBytesConfig.to_json_file.json_file_path",description:`<strong>json_file_path</strong> (<code>str</code> or <code>os.PathLike</code>) &#x2014;
Path to the JSON file in which this configuration instance&#x2019;s parameters will be saved.`,name:"json_file_path"},{anchor:"transformers.BitsAndBytesConfig.to_json_file.use_diff",description:`<strong>use_diff</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If set to <code>True</code>, only the difference between the config instance and the default
<code>BitsAndBytesConfig()</code> is serialized to JSON file.`,name:"use_diff"}],source:"https://github.com/huggingface/transformers/blob/v4.29.0/src/transformers/utils/quantization_config.py#L125"}}),st=new T({}),{c(){v=r("meta"),z=h(),y=r("h1"),$=r("a"),Et=r("span"),m(Ce.$$.fragment),Za=h(),At=r("span"),es=a("Quantize \u{1F917} Transformers models"),Ho=h(),U=r("h2"),te=r("a"),kt=r("span"),m(ze.$$.fragment),ts=h(),lt=r("span"),Ct=r("code"),os=a("bitsandbytes"),as=a(" Integration"),Uo=h(),q=r("p"),ss=a("\u{1F917} Transformers is closely integrated with most used modules on "),zt=r("code"),rs=a("bitsandbytes"),ns=a(`. You can load your model in 8-bit precision with few lines of code.
This is supported by most of the GPU hardwares since the `),qt=r("code"),is=a("0.37.0"),ls=a(" release of "),Pt=r("code"),ds=a("bitsandbytes"),hs=a("."),Io=h(),D=r("p"),fs=a("Learn more about the quantization method in the "),qe=r("a"),us=a("LLM.int8()"),ms=a(" paper, or the "),Pe=r("a"),cs=a("blogpost"),ps=a(" about the collaboration."),Go=h(),oe=r("p"),_s=a("Here are the things you can do using "),Bt=r("code"),bs=a("bitsandbytes"),gs=a(" integration"),Yo=h(),I=r("h3"),ae=r("a"),Tt=r("span"),m(Be.$$.fragment),vs=h(),Mt=r("span"),ys=a("Load a large model in 8bit"),Jo=h(),x=r("p"),ws=a("You can load a model by roughly halving the memory requirements by using "),Dt=r("code"),$s=a("load_in_8bit=True"),Es=a(" argument when calling "),xt=r("code"),As=a(".from_pretrained"),ks=a(" method"),Vo=h(),m(Te.$$.fragment),Qo=h(),se=r("p"),Cs=a("Then, use your model as you would usually use a "),dt=r("a"),zs=a("PreTrainedModel"),qs=a("."),Wo=h(),re=r("p"),Ps=a("You can check the memory footprint of your model with "),Lt=r("code"),Bs=a("get_memory_footprint"),Ts=a(" method."),Ro=h(),m(Me.$$.fragment),Ko=h(),ht=r("p"),Ms=a("With this integration we were able to load large models on smaller devices and run them without any issue."),Xo=h(),m(ne.$$.fragment),Zo=h(),G=r("h3"),ie=r("a"),jt=r("span"),m(De.$$.fragment),Ds=h(),Ot=r("span"),xs=a("Push quantized models on the \u{1F917} Hub"),ea=h(),P=r("p"),Ls=a("You can push a quantized model on the Hub by naively using "),St=r("code"),js=a("push_to_hub"),Os=a(` method. This will first push the quantization configuration file, then push the quantized model weights.
Make sure to use `),Nt=r("code"),Ss=a("bitsandbytes>0.37.2"),Ns=a(" (at this time of writing, we tested it on "),Ft=r("code"),Fs=a("bitsandbytes==0.38.0.post1"),Hs=a(") to be able to use this feature."),ta=h(),m(xe.$$.fragment),oa=h(),m(le.$$.fragment),aa=h(),Y=r("h3"),de=r("a"),Ht=r("span"),m(Le.$$.fragment),Us=h(),Ut=r("span"),Is=a("Load a quantized model from the \u{1F917} Hub"),sa=h(),L=r("p"),Gs=a("You can load a quantized model from the Hub by using "),It=r("code"),Ys=a("from_pretrained"),Js=a(" method. Make sure that the pushed weights are quantized, by checking that the attribute "),Gt=r("code"),Vs=a("quantization_config"),Qs=a(" is present in the model configuration object."),ra=h(),m(je.$$.fragment),na=h(),A=r("p"),Ws=a("Note that in this case, you don\u2019t need to specify the arguments "),Yt=r("code"),Rs=a("load_in_8bit=True"),Ks=a(" and "),Jt=r("code"),Xs=a('device_map="auto"'),Zs=a(", but you need to make sure that "),Vt=r("code"),er=a("bitsandbytes"),tr=a(" and "),Qt=r("code"),or=a("accelerate"),ar=a(" are installed."),ia=h(),J=r("h3"),he=r("a"),Wt=r("span"),m(Oe.$$.fragment),sr=h(),Rt=r("span"),rr=a("Advanced usecases"),la=h(),ft=r("p"),nr=a("This section is intended to advanced users, that want to explore what it is possible to do beyond loading and running 8-bit models."),da=h(),V=r("h4"),fe=r("a"),Kt=r("span"),m(Se.$$.fragment),ir=h(),ue=r("span"),lr=a("Offload between "),Xt=r("code"),dr=a("cpu"),hr=a(" and "),Zt=r("code"),fr=a("gpu"),ha=h(),k=r("p"),ur=a("One of the advanced usecase of this is being able to load a model and dispatch the weights between "),eo=r("code"),mr=a("CPU"),cr=a(" and "),to=r("code"),pr=a("GPU"),_r=a(". Note that the weights that will be dispatched on CPU "),oo=r("strong"),br=a("will not"),gr=a(" be converted in 8-bit, thus kept in "),ao=r("code"),vr=a("float32"),yr=a(". This feature is intended for users that want to fit a very large model and dispatch the model between GPU and CPU."),fa=h(),C=r("p"),wr=a("First, load a "),so=r("code"),$r=a("BitsAndBytesConfig"),Er=a(" from "),ro=r("code"),Ar=a("transformers"),kr=a(" and set the attribute "),no=r("code"),Cr=a("llm_int8_enable_fp32_cpu_offload"),zr=a(" to "),io=r("code"),qr=a("True"),Pr=a(":"),ua=h(),m(Ne.$$.fragment),ma=h(),j=r("p"),Br=a("Let\u2019s say you want to load "),lo=r("code"),Tr=a("bigscience/bloom-1b7"),Mr=a(" model, and you have just enough GPU RAM to fit the entire model except the "),ho=r("code"),Dr=a("lm_head"),xr=a(". Therefore write a custom device_map as follows:"),ca=h(),m(Fe.$$.fragment),pa=h(),ut=r("p"),Lr=a("And load your model as follows:"),_a=h(),m(He.$$.fragment),ba=h(),mt=r("p"),jr=a("And that\u2019s it! Enjoy your model!"),ga=h(),Q=r("h4"),me=r("a"),fo=r("span"),m(Ue.$$.fragment),Or=h(),ct=r("span"),Sr=a("Play with "),uo=r("code"),Nr=a("llm_int8_threshold"),va=h(),O=r("p"),Fr=a("You can play with the "),mo=r("code"),Hr=a("llm_int8_threshold"),Ur=a(` argument to change the threshold of the outliers. An \u201Coutlier\u201D is a hidden state value that is greater than a certain threshold.
This corresponds to the outlier threshold for outlier detection as described in `),co=r("code"),Ir=a("LLM.int8()"),Gr=a(` paper. Any hidden states value that is above this threshold will be considered an outlier and the operation on those values will be done in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but there are some exceptional systematic outliers that are very differently distributed for large models. These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6, but a lower threshold might be needed for more unstable models (small models, fine-tuning).
This argument can impact the inference speed of the model. We suggest to play with this parameter to find which one is the best for your usecase.`),ya=h(),m(Ie.$$.fragment),wa=h(),W=r("h4"),ce=r("a"),po=r("span"),m(Ge.$$.fragment),Yr=h(),_o=r("span"),Jr=a("Skip the conversion of some modules"),$a=h(),R=r("p"),Vr=a("Some models has several modules that needs to be not converted in 8-bit to ensure stability. For example Jukebox model has several "),bo=r("code"),Qr=a("lm_head"),Wr=a(" modules that should be skipped. Play with "),go=r("code"),Rr=a("llm_int8_skip_modules"),Ea=h(),m(Ye.$$.fragment),Aa=h(),K=r("h4"),pe=r("a"),vo=r("span"),m(Je.$$.fragment),Kr=h(),yo=r("span"),Xr=a("Fine-tune a model that has been loaded in 8-bit"),ka=h(),B=r("p"),Zr=a(`With the official support of adapters in the Hugging Face ecosystem, you can fine-tune models that have been loaded in 8-bit.
This enables fine-tuning large models such as `),wo=r("code"),en=a("flan-t5-large"),tn=a(" or "),$o=r("code"),on=a("facebook/opt-6.7b"),an=a(" in a single google Colab. Please have a look at "),Ve=r("a"),Eo=r("code"),sn=a("peft"),rn=a(" library for more details."),Ca=h(),X=r("h3"),_e=r("a"),Ao=r("span"),m(Qe.$$.fragment),nn=h(),ko=r("span"),ln=a("BitsAndBytesConfig"),za=h(),w=r("div"),m(We.$$.fragment),dn=h(),Re=r("p"),hn=a(`This is a wrapper class about all possible attributes and features that you can play with a model that has been
loaded using `),Co=r("code"),fn=a("bitsandbytes"),un=a("."),mn=h(),Ke=r("p"),cn=a("This replaces "),zo=r("code"),pn=a("load_in_8bit"),_n=a(" therefore both options are mutually exclusive."),bn=h(),M=r("p"),gn=a("For now, only arguments that are relative to "),qo=r("code"),vn=a("LLM.int8()"),yn=a(` are supported, therefore the arguments are all termed as
`),Po=r("code"),wn=a("llm_int8_*"),$n=a(". If more methods are added to "),Bo=r("code"),En=a("bitsandbytes"),An=a(", then more arguments will be added to this class."),kn=h(),be=r("div"),m(Xe.$$.fragment),Cn=h(),Ze=r("p"),zn=a("Instantiates a "),pt=r("a"),qn=a("BitsAndBytesConfig"),Pn=a(" from a Python dictionary of parameters."),Bn=h(),ge=r("div"),m(et.$$.fragment),Tn=h(),To=r("p"),Mn=a("Safety checker that arguments are correct - also replaces some NoneType arguments with their default values."),Dn=h(),ve=r("div"),m(tt.$$.fragment),xn=h(),ot=r("p"),Ln=a(`Serializes this instance to a Python dictionary. Returns:
`),Mo=r("code"),jn=a("Dict[str, Any]"),On=a(": Dictionary of all the attributes that make up this configuration instance."),Sn=h(),ye=r("div"),m(at.$$.fragment),Nn=h(),Do=r("p"),Fn=a("Save this instance to a JSON file."),qa=h(),Z=r("h2"),we=r("a"),xo=r("span"),m(st.$$.fragment),Hn=h(),_t=r("span"),Un=a("Quantization with \u{1F917} "),Lo=r("code"),In=a("optimum"),Pa=h(),S=r("p"),Gn=a("Please have a look at "),rt=r("a"),Yn=a("Optimum documentation"),Jn=a(" to learn more about quantization methods that are supported by "),jo=r("code"),Vn=a("optimum"),Qn=a(" and see if these are applicable for your usecase."),this.h()},l(e){const l=jl('[data-svelte="svelte-1phssyn"]',document.head);v=n(l,"META",{name:!0,content:!0}),l.forEach(o),z=f(e),y=n(e,"H1",{class:!0});var nt=i(y);$=n(nt,"A",{id:!0,class:!0,href:!0});var Oo=i($);Et=n(Oo,"SPAN",{});var Xn=i(Et);c(Ce.$$.fragment,Xn),Xn.forEach(o),Oo.forEach(o),Za=f(nt),At=n(nt,"SPAN",{});var Zn=i(At);es=s(Zn,"Quantize \u{1F917} Transformers models"),Zn.forEach(o),nt.forEach(o),Ho=f(e),U=n(e,"H2",{class:!0});var Ta=i(U);te=n(Ta,"A",{id:!0,class:!0,href:!0});var ei=i(te);kt=n(ei,"SPAN",{});var ti=i(kt);c(ze.$$.fragment,ti),ti.forEach(o),ei.forEach(o),ts=f(Ta),lt=n(Ta,"SPAN",{});var Wn=i(lt);Ct=n(Wn,"CODE",{});var oi=i(Ct);os=s(oi,"bitsandbytes"),oi.forEach(o),as=s(Wn," Integration"),Wn.forEach(o),Ta.forEach(o),Uo=f(e),q=n(e,"P",{});var $e=i(q);ss=s($e,"\u{1F917} Transformers is closely integrated with most used modules on "),zt=n($e,"CODE",{});var ai=i(zt);rs=s(ai,"bitsandbytes"),ai.forEach(o),ns=s($e,`. You can load your model in 8-bit precision with few lines of code.
This is supported by most of the GPU hardwares since the `),qt=n($e,"CODE",{});var si=i(qt);is=s(si,"0.37.0"),si.forEach(o),ls=s($e," release of "),Pt=n($e,"CODE",{});var ri=i(Pt);ds=s(ri,"bitsandbytes"),ri.forEach(o),hs=s($e,"."),$e.forEach(o),Io=f(e),D=n(e,"P",{});var bt=i(D);fs=s(bt,"Learn more about the quantization method in the "),qe=n(bt,"A",{href:!0,rel:!0});var ni=i(qe);us=s(ni,"LLM.int8()"),ni.forEach(o),ms=s(bt," paper, or the "),Pe=n(bt,"A",{href:!0,rel:!0});var ii=i(Pe);cs=s(ii,"blogpost"),ii.forEach(o),ps=s(bt," about the collaboration."),bt.forEach(o),Go=f(e),oe=n(e,"P",{});var Ma=i(oe);_s=s(Ma,"Here are the things you can do using "),Bt=n(Ma,"CODE",{});var li=i(Bt);bs=s(li,"bitsandbytes"),li.forEach(o),gs=s(Ma," integration"),Ma.forEach(o),Yo=f(e),I=n(e,"H3",{class:!0});var Da=i(I);ae=n(Da,"A",{id:!0,class:!0,href:!0});var di=i(ae);Tt=n(di,"SPAN",{});var hi=i(Tt);c(Be.$$.fragment,hi),hi.forEach(o),di.forEach(o),vs=f(Da),Mt=n(Da,"SPAN",{});var fi=i(Mt);ys=s(fi,"Load a large model in 8bit"),fi.forEach(o),Da.forEach(o),Jo=f(e),x=n(e,"P",{});var gt=i(x);ws=s(gt,"You can load a model by roughly halving the memory requirements by using "),Dt=n(gt,"CODE",{});var ui=i(Dt);$s=s(ui,"load_in_8bit=True"),ui.forEach(o),Es=s(gt," argument when calling "),xt=n(gt,"CODE",{});var mi=i(xt);As=s(mi,".from_pretrained"),mi.forEach(o),ks=s(gt," method"),gt.forEach(o),Vo=f(e),c(Te.$$.fragment,e),Qo=f(e),se=n(e,"P",{});var xa=i(se);Cs=s(xa,"Then, use your model as you would usually use a "),dt=n(xa,"A",{href:!0});var ci=i(dt);zs=s(ci,"PreTrainedModel"),ci.forEach(o),qs=s(xa,"."),xa.forEach(o),Wo=f(e),re=n(e,"P",{});var La=i(re);Ps=s(La,"You can check the memory footprint of your model with "),Lt=n(La,"CODE",{});var pi=i(Lt);Bs=s(pi,"get_memory_footprint"),pi.forEach(o),Ts=s(La," method."),La.forEach(o),Ro=f(e),c(Me.$$.fragment,e),Ko=f(e),ht=n(e,"P",{});var _i=i(ht);Ms=s(_i,"With this integration we were able to load large models on smaller devices and run them without any issue."),_i.forEach(o),Xo=f(e),c(ne.$$.fragment,e),Zo=f(e),G=n(e,"H3",{class:!0});var ja=i(G);ie=n(ja,"A",{id:!0,class:!0,href:!0});var bi=i(ie);jt=n(bi,"SPAN",{});var gi=i(jt);c(De.$$.fragment,gi),gi.forEach(o),bi.forEach(o),Ds=f(ja),Ot=n(ja,"SPAN",{});var vi=i(Ot);xs=s(vi,"Push quantized models on the \u{1F917} Hub"),vi.forEach(o),ja.forEach(o),ea=f(e),P=n(e,"P",{});var Ee=i(P);Ls=s(Ee,"You can push a quantized model on the Hub by naively using "),St=n(Ee,"CODE",{});var yi=i(St);js=s(yi,"push_to_hub"),yi.forEach(o),Os=s(Ee,` method. This will first push the quantization configuration file, then push the quantized model weights.
Make sure to use `),Nt=n(Ee,"CODE",{});var wi=i(Nt);Ss=s(wi,"bitsandbytes>0.37.2"),wi.forEach(o),Ns=s(Ee," (at this time of writing, we tested it on "),Ft=n(Ee,"CODE",{});var $i=i(Ft);Fs=s($i,"bitsandbytes==0.38.0.post1"),$i.forEach(o),Hs=s(Ee,") to be able to use this feature."),Ee.forEach(o),ta=f(e),c(xe.$$.fragment,e),oa=f(e),c(le.$$.fragment,e),aa=f(e),Y=n(e,"H3",{class:!0});var Oa=i(Y);de=n(Oa,"A",{id:!0,class:!0,href:!0});var Ei=i(de);Ht=n(Ei,"SPAN",{});var Ai=i(Ht);c(Le.$$.fragment,Ai),Ai.forEach(o),Ei.forEach(o),Us=f(Oa),Ut=n(Oa,"SPAN",{});var ki=i(Ut);Is=s(ki,"Load a quantized model from the \u{1F917} Hub"),ki.forEach(o),Oa.forEach(o),sa=f(e),L=n(e,"P",{});var vt=i(L);Gs=s(vt,"You can load a quantized model from the Hub by using "),It=n(vt,"CODE",{});var Ci=i(It);Ys=s(Ci,"from_pretrained"),Ci.forEach(o),Js=s(vt," method. Make sure that the pushed weights are quantized, by checking that the attribute "),Gt=n(vt,"CODE",{});var zi=i(Gt);Vs=s(zi,"quantization_config"),zi.forEach(o),Qs=s(vt," is present in the model configuration object."),vt.forEach(o),ra=f(e),c(je.$$.fragment,e),na=f(e),A=n(e,"P",{});var N=i(A);Ws=s(N,"Note that in this case, you don\u2019t need to specify the arguments "),Yt=n(N,"CODE",{});var qi=i(Yt);Rs=s(qi,"load_in_8bit=True"),qi.forEach(o),Ks=s(N," and "),Jt=n(N,"CODE",{});var Pi=i(Jt);Xs=s(Pi,'device_map="auto"'),Pi.forEach(o),Zs=s(N,", but you need to make sure that "),Vt=n(N,"CODE",{});var Bi=i(Vt);er=s(Bi,"bitsandbytes"),Bi.forEach(o),tr=s(N," and "),Qt=n(N,"CODE",{});var Ti=i(Qt);or=s(Ti,"accelerate"),Ti.forEach(o),ar=s(N," are installed."),N.forEach(o),ia=f(e),J=n(e,"H3",{class:!0});var Sa=i(J);he=n(Sa,"A",{id:!0,class:!0,href:!0});var Mi=i(he);Wt=n(Mi,"SPAN",{});var Di=i(Wt);c(Oe.$$.fragment,Di),Di.forEach(o),Mi.forEach(o),sr=f(Sa),Rt=n(Sa,"SPAN",{});var xi=i(Rt);rr=s(xi,"Advanced usecases"),xi.forEach(o),Sa.forEach(o),la=f(e),ft=n(e,"P",{});var Li=i(ft);nr=s(Li,"This section is intended to advanced users, that want to explore what it is possible to do beyond loading and running 8-bit models."),Li.forEach(o),da=f(e),V=n(e,"H4",{class:!0});var Na=i(V);fe=n(Na,"A",{id:!0,class:!0,href:!0});var ji=i(fe);Kt=n(ji,"SPAN",{});var Oi=i(Kt);c(Se.$$.fragment,Oi),Oi.forEach(o),ji.forEach(o),ir=f(Na),ue=n(Na,"SPAN",{});var So=i(ue);lr=s(So,"Offload between "),Xt=n(So,"CODE",{});var Si=i(Xt);dr=s(Si,"cpu"),Si.forEach(o),hr=s(So," and "),Zt=n(So,"CODE",{});var Ni=i(Zt);fr=s(Ni,"gpu"),Ni.forEach(o),So.forEach(o),Na.forEach(o),ha=f(e),k=n(e,"P",{});var F=i(k);ur=s(F,"One of the advanced usecase of this is being able to load a model and dispatch the weights between "),eo=n(F,"CODE",{});var Fi=i(eo);mr=s(Fi,"CPU"),Fi.forEach(o),cr=s(F," and "),to=n(F,"CODE",{});var Hi=i(to);pr=s(Hi,"GPU"),Hi.forEach(o),_r=s(F,". Note that the weights that will be dispatched on CPU "),oo=n(F,"STRONG",{});var Ui=i(oo);br=s(Ui,"will not"),Ui.forEach(o),gr=s(F," be converted in 8-bit, thus kept in "),ao=n(F,"CODE",{});var Ii=i(ao);vr=s(Ii,"float32"),Ii.forEach(o),yr=s(F,". This feature is intended for users that want to fit a very large model and dispatch the model between GPU and CPU."),F.forEach(o),fa=f(e),C=n(e,"P",{});var H=i(C);wr=s(H,"First, load a "),so=n(H,"CODE",{});var Gi=i(so);$r=s(Gi,"BitsAndBytesConfig"),Gi.forEach(o),Er=s(H," from "),ro=n(H,"CODE",{});var Yi=i(ro);Ar=s(Yi,"transformers"),Yi.forEach(o),kr=s(H," and set the attribute "),no=n(H,"CODE",{});var Ji=i(no);Cr=s(Ji,"llm_int8_enable_fp32_cpu_offload"),Ji.forEach(o),zr=s(H," to "),io=n(H,"CODE",{});var Vi=i(io);qr=s(Vi,"True"),Vi.forEach(o),Pr=s(H,":"),H.forEach(o),ua=f(e),c(Ne.$$.fragment,e),ma=f(e),j=n(e,"P",{});var yt=i(j);Br=s(yt,"Let\u2019s say you want to load "),lo=n(yt,"CODE",{});var Qi=i(lo);Tr=s(Qi,"bigscience/bloom-1b7"),Qi.forEach(o),Mr=s(yt," model, and you have just enough GPU RAM to fit the entire model except the "),ho=n(yt,"CODE",{});var Wi=i(ho);Dr=s(Wi,"lm_head"),Wi.forEach(o),xr=s(yt,". Therefore write a custom device_map as follows:"),yt.forEach(o),ca=f(e),c(Fe.$$.fragment,e),pa=f(e),ut=n(e,"P",{});var Ri=i(ut);Lr=s(Ri,"And load your model as follows:"),Ri.forEach(o),_a=f(e),c(He.$$.fragment,e),ba=f(e),mt=n(e,"P",{});var Ki=i(mt);jr=s(Ki,"And that\u2019s it! Enjoy your model!"),Ki.forEach(o),ga=f(e),Q=n(e,"H4",{class:!0});var Fa=i(Q);me=n(Fa,"A",{id:!0,class:!0,href:!0});var Xi=i(me);fo=n(Xi,"SPAN",{});var Zi=i(fo);c(Ue.$$.fragment,Zi),Zi.forEach(o),Xi.forEach(o),Or=f(Fa),ct=n(Fa,"SPAN",{});var Rn=i(ct);Sr=s(Rn,"Play with "),uo=n(Rn,"CODE",{});var el=i(uo);Nr=s(el,"llm_int8_threshold"),el.forEach(o),Rn.forEach(o),Fa.forEach(o),va=f(e),O=n(e,"P",{});var wt=i(O);Fr=s(wt,"You can play with the "),mo=n(wt,"CODE",{});var tl=i(mo);Hr=s(tl,"llm_int8_threshold"),tl.forEach(o),Ur=s(wt,` argument to change the threshold of the outliers. An \u201Coutlier\u201D is a hidden state value that is greater than a certain threshold.
This corresponds to the outlier threshold for outlier detection as described in `),co=n(wt,"CODE",{});var ol=i(co);Ir=s(ol,"LLM.int8()"),ol.forEach(o),Gr=s(wt,` paper. Any hidden states value that is above this threshold will be considered an outlier and the operation on those values will be done in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but there are some exceptional systematic outliers that are very differently distributed for large models. These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6, but a lower threshold might be needed for more unstable models (small models, fine-tuning).
This argument can impact the inference speed of the model. We suggest to play with this parameter to find which one is the best for your usecase.`),wt.forEach(o),ya=f(e),c(Ie.$$.fragment,e),wa=f(e),W=n(e,"H4",{class:!0});var Ha=i(W);ce=n(Ha,"A",{id:!0,class:!0,href:!0});var al=i(ce);po=n(al,"SPAN",{});var sl=i(po);c(Ge.$$.fragment,sl),sl.forEach(o),al.forEach(o),Yr=f(Ha),_o=n(Ha,"SPAN",{});var rl=i(_o);Jr=s(rl,"Skip the conversion of some modules"),rl.forEach(o),Ha.forEach(o),$a=f(e),R=n(e,"P",{});var No=i(R);Vr=s(No,"Some models has several modules that needs to be not converted in 8-bit to ensure stability. For example Jukebox model has several "),bo=n(No,"CODE",{});var nl=i(bo);Qr=s(nl,"lm_head"),nl.forEach(o),Wr=s(No," modules that should be skipped. Play with "),go=n(No,"CODE",{});var il=i(go);Rr=s(il,"llm_int8_skip_modules"),il.forEach(o),No.forEach(o),Ea=f(e),c(Ye.$$.fragment,e),Aa=f(e),K=n(e,"H4",{class:!0});var Ua=i(K);pe=n(Ua,"A",{id:!0,class:!0,href:!0});var ll=i(pe);vo=n(ll,"SPAN",{});var dl=i(vo);c(Je.$$.fragment,dl),dl.forEach(o),ll.forEach(o),Kr=f(Ua),yo=n(Ua,"SPAN",{});var hl=i(yo);Xr=s(hl,"Fine-tune a model that has been loaded in 8-bit"),hl.forEach(o),Ua.forEach(o),ka=f(e),B=n(e,"P",{});var Ae=i(B);Zr=s(Ae,`With the official support of adapters in the Hugging Face ecosystem, you can fine-tune models that have been loaded in 8-bit.
This enables fine-tuning large models such as `),wo=n(Ae,"CODE",{});var fl=i(wo);en=s(fl,"flan-t5-large"),fl.forEach(o),tn=s(Ae," or "),$o=n(Ae,"CODE",{});var ul=i($o);on=s(ul,"facebook/opt-6.7b"),ul.forEach(o),an=s(Ae," in a single google Colab. Please have a look at "),Ve=n(Ae,"A",{href:!0,rel:!0});var ml=i(Ve);Eo=n(ml,"CODE",{});var cl=i(Eo);sn=s(cl,"peft"),cl.forEach(o),ml.forEach(o),rn=s(Ae," library for more details."),Ae.forEach(o),Ca=f(e),X=n(e,"H3",{class:!0});var Ia=i(X);_e=n(Ia,"A",{id:!0,class:!0,href:!0});var pl=i(_e);Ao=n(pl,"SPAN",{});var _l=i(Ao);c(Qe.$$.fragment,_l),_l.forEach(o),pl.forEach(o),nn=f(Ia),ko=n(Ia,"SPAN",{});var bl=i(ko);ln=s(bl,"BitsAndBytesConfig"),bl.forEach(o),Ia.forEach(o),za=f(e),w=n(e,"DIV",{class:!0});var E=i(w);c(We.$$.fragment,E),dn=f(E),Re=n(E,"P",{});var Ga=i(Re);hn=s(Ga,`This is a wrapper class about all possible attributes and features that you can play with a model that has been
loaded using `),Co=n(Ga,"CODE",{});var gl=i(Co);fn=s(gl,"bitsandbytes"),gl.forEach(o),un=s(Ga,"."),Ga.forEach(o),mn=f(E),Ke=n(E,"P",{});var Ya=i(Ke);cn=s(Ya,"This replaces "),zo=n(Ya,"CODE",{});var vl=i(zo);pn=s(vl,"load_in_8bit"),vl.forEach(o),_n=s(Ya," therefore both options are mutually exclusive."),Ya.forEach(o),bn=f(E),M=n(E,"P",{});var ke=i(M);gn=s(ke,"For now, only arguments that are relative to "),qo=n(ke,"CODE",{});var yl=i(qo);vn=s(yl,"LLM.int8()"),yl.forEach(o),yn=s(ke,` are supported, therefore the arguments are all termed as
`),Po=n(ke,"CODE",{});var wl=i(Po);wn=s(wl,"llm_int8_*"),wl.forEach(o),$n=s(ke,". If more methods are added to "),Bo=n(ke,"CODE",{});var $l=i(Bo);En=s($l,"bitsandbytes"),$l.forEach(o),An=s(ke,", then more arguments will be added to this class."),ke.forEach(o),kn=f(E),be=n(E,"DIV",{class:!0});var Ja=i(be);c(Xe.$$.fragment,Ja),Cn=f(Ja),Ze=n(Ja,"P",{});var Va=i(Ze);zn=s(Va,"Instantiates a "),pt=n(Va,"A",{href:!0});var El=i(pt);qn=s(El,"BitsAndBytesConfig"),El.forEach(o),Pn=s(Va," from a Python dictionary of parameters."),Va.forEach(o),Ja.forEach(o),Bn=f(E),ge=n(E,"DIV",{class:!0});var Qa=i(ge);c(et.$$.fragment,Qa),Tn=f(Qa),To=n(Qa,"P",{});var Al=i(To);Mn=s(Al,"Safety checker that arguments are correct - also replaces some NoneType arguments with their default values."),Al.forEach(o),Qa.forEach(o),Dn=f(E),ve=n(E,"DIV",{class:!0});var Wa=i(ve);c(tt.$$.fragment,Wa),xn=f(Wa),ot=n(Wa,"P",{});var Ra=i(ot);Ln=s(Ra,`Serializes this instance to a Python dictionary. Returns:
`),Mo=n(Ra,"CODE",{});var kl=i(Mo);jn=s(kl,"Dict[str, Any]"),kl.forEach(o),On=s(Ra,": Dictionary of all the attributes that make up this configuration instance."),Ra.forEach(o),Wa.forEach(o),Sn=f(E),ye=n(E,"DIV",{class:!0});var Ka=i(ye);c(at.$$.fragment,Ka),Nn=f(Ka),Do=n(Ka,"P",{});var Cl=i(Do);Fn=s(Cl,"Save this instance to a JSON file."),Cl.forEach(o),Ka.forEach(o),E.forEach(o),qa=f(e),Z=n(e,"H2",{class:!0});var Xa=i(Z);we=n(Xa,"A",{id:!0,class:!0,href:!0});var zl=i(we);xo=n(zl,"SPAN",{});var ql=i(xo);c(st.$$.fragment,ql),ql.forEach(o),zl.forEach(o),Hn=f(Xa),_t=n(Xa,"SPAN",{});var Kn=i(_t);Un=s(Kn,"Quantization with \u{1F917} "),Lo=n(Kn,"CODE",{});var Pl=i(Lo);In=s(Pl,"optimum"),Pl.forEach(o),Kn.forEach(o),Xa.forEach(o),Pa=f(e),S=n(e,"P",{});var $t=i(S);Gn=s($t,"Please have a look at "),rt=n($t,"A",{href:!0,rel:!0});var Bl=i(rt);Yn=s(Bl,"Optimum documentation"),Bl.forEach(o),Jn=s($t," to learn more about quantization methods that are supported by "),jo=n($t,"CODE",{});var Tl=i(jo);Vn=s(Tl,"optimum"),Tl.forEach(o),Qn=s($t," and see if these are applicable for your usecase."),$t.forEach(o),this.h()},h(){u(v,"name","hf:doc:metadata"),u(v,"content",JSON.stringify(Hl)),u($,"id","quantize-transformers-models"),u($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u($,"href","#quantize-transformers-models"),u(y,"class","relative group"),u(te,"id","bitsandbytes-integration"),u(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(te,"href","#bitsandbytes-integration"),u(U,"class","relative group"),u(qe,"href","https://arxiv.org/abs/2208.07339"),u(qe,"rel","nofollow"),u(Pe,"href","https://huggingface.co/blog/hf-bitsandbytes-integration"),u(Pe,"rel","nofollow"),u(ae,"id","load-a-large-model-in-8bit"),u(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ae,"href","#load-a-large-model-in-8bit"),u(I,"class","relative group"),u(dt,"href","/docs/transformers/v4.29.0/en/main_classes/model#transformers.PreTrainedModel"),u(ie,"id","push-quantized-models-on-the-hub"),u(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ie,"href","#push-quantized-models-on-the-hub"),u(G,"class","relative group"),u(de,"id","load-a-quantized-model-from-the-hub"),u(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(de,"href","#load-a-quantized-model-from-the-hub"),u(Y,"class","relative group"),u(he,"id","advanced-usecases"),u(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(he,"href","#advanced-usecases"),u(J,"class","relative group"),u(fe,"id","offload-between-cpu-and-gpu"),u(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(fe,"href","#offload-between-cpu-and-gpu"),u(V,"class","relative group"),u(me,"id","play-with-llmint8threshold"),u(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(me,"href","#play-with-llmint8threshold"),u(Q,"class","relative group"),u(ce,"id","skip-the-conversion-of-some-modules"),u(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ce,"href","#skip-the-conversion-of-some-modules"),u(W,"class","relative group"),u(pe,"id","finetune-a-model-that-has-been-loaded-in-8bit"),u(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(pe,"href","#finetune-a-model-that-has-been-loaded-in-8bit"),u(K,"class","relative group"),u(Ve,"href","https://github.com/huggingface/peft"),u(Ve,"rel","nofollow"),u(_e,"id","transformers.BitsAndBytesConfig"),u(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(_e,"href","#transformers.BitsAndBytesConfig"),u(X,"class","relative group"),u(pt,"href","/docs/transformers/v4.29.0/en/main_classes/quantization#transformers.BitsAndBytesConfig"),u(be,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(ge,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(ve,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(ye,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(we,"id","quantization-with-optimum"),u(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(we,"href","#quantization-with-optimum"),u(Z,"class","relative group"),u(rt,"href","https://huggingface.co/docs/optimum/index"),u(rt,"rel","nofollow")},m(e,l){t(document.head,v),d(e,z,l),d(e,y,l),t(y,$),t($,Et),p(Ce,Et,null),t(y,Za),t(y,At),t(At,es),d(e,Ho,l),d(e,U,l),t(U,te),t(te,kt),p(ze,kt,null),t(U,ts),t(U,lt),t(lt,Ct),t(Ct,os),t(lt,as),d(e,Uo,l),d(e,q,l),t(q,ss),t(q,zt),t(zt,rs),t(q,ns),t(q,qt),t(qt,is),t(q,ls),t(q,Pt),t(Pt,ds),t(q,hs),d(e,Io,l),d(e,D,l),t(D,fs),t(D,qe),t(qe,us),t(D,ms),t(D,Pe),t(Pe,cs),t(D,ps),d(e,Go,l),d(e,oe,l),t(oe,_s),t(oe,Bt),t(Bt,bs),t(oe,gs),d(e,Yo,l),d(e,I,l),t(I,ae),t(ae,Tt),p(Be,Tt,null),t(I,vs),t(I,Mt),t(Mt,ys),d(e,Jo,l),d(e,x,l),t(x,ws),t(x,Dt),t(Dt,$s),t(x,Es),t(x,xt),t(xt,As),t(x,ks),d(e,Vo,l),p(Te,e,l),d(e,Qo,l),d(e,se,l),t(se,Cs),t(se,dt),t(dt,zs),t(se,qs),d(e,Wo,l),d(e,re,l),t(re,Ps),t(re,Lt),t(Lt,Bs),t(re,Ts),d(e,Ro,l),p(Me,e,l),d(e,Ko,l),d(e,ht,l),t(ht,Ms),d(e,Xo,l),p(ne,e,l),d(e,Zo,l),d(e,G,l),t(G,ie),t(ie,jt),p(De,jt,null),t(G,Ds),t(G,Ot),t(Ot,xs),d(e,ea,l),d(e,P,l),t(P,Ls),t(P,St),t(St,js),t(P,Os),t(P,Nt),t(Nt,Ss),t(P,Ns),t(P,Ft),t(Ft,Fs),t(P,Hs),d(e,ta,l),p(xe,e,l),d(e,oa,l),p(le,e,l),d(e,aa,l),d(e,Y,l),t(Y,de),t(de,Ht),p(Le,Ht,null),t(Y,Us),t(Y,Ut),t(Ut,Is),d(e,sa,l),d(e,L,l),t(L,Gs),t(L,It),t(It,Ys),t(L,Js),t(L,Gt),t(Gt,Vs),t(L,Qs),d(e,ra,l),p(je,e,l),d(e,na,l),d(e,A,l),t(A,Ws),t(A,Yt),t(Yt,Rs),t(A,Ks),t(A,Jt),t(Jt,Xs),t(A,Zs),t(A,Vt),t(Vt,er),t(A,tr),t(A,Qt),t(Qt,or),t(A,ar),d(e,ia,l),d(e,J,l),t(J,he),t(he,Wt),p(Oe,Wt,null),t(J,sr),t(J,Rt),t(Rt,rr),d(e,la,l),d(e,ft,l),t(ft,nr),d(e,da,l),d(e,V,l),t(V,fe),t(fe,Kt),p(Se,Kt,null),t(V,ir),t(V,ue),t(ue,lr),t(ue,Xt),t(Xt,dr),t(ue,hr),t(ue,Zt),t(Zt,fr),d(e,ha,l),d(e,k,l),t(k,ur),t(k,eo),t(eo,mr),t(k,cr),t(k,to),t(to,pr),t(k,_r),t(k,oo),t(oo,br),t(k,gr),t(k,ao),t(ao,vr),t(k,yr),d(e,fa,l),d(e,C,l),t(C,wr),t(C,so),t(so,$r),t(C,Er),t(C,ro),t(ro,Ar),t(C,kr),t(C,no),t(no,Cr),t(C,zr),t(C,io),t(io,qr),t(C,Pr),d(e,ua,l),p(Ne,e,l),d(e,ma,l),d(e,j,l),t(j,Br),t(j,lo),t(lo,Tr),t(j,Mr),t(j,ho),t(ho,Dr),t(j,xr),d(e,ca,l),p(Fe,e,l),d(e,pa,l),d(e,ut,l),t(ut,Lr),d(e,_a,l),p(He,e,l),d(e,ba,l),d(e,mt,l),t(mt,jr),d(e,ga,l),d(e,Q,l),t(Q,me),t(me,fo),p(Ue,fo,null),t(Q,Or),t(Q,ct),t(ct,Sr),t(ct,uo),t(uo,Nr),d(e,va,l),d(e,O,l),t(O,Fr),t(O,mo),t(mo,Hr),t(O,Ur),t(O,co),t(co,Ir),t(O,Gr),d(e,ya,l),p(Ie,e,l),d(e,wa,l),d(e,W,l),t(W,ce),t(ce,po),p(Ge,po,null),t(W,Yr),t(W,_o),t(_o,Jr),d(e,$a,l),d(e,R,l),t(R,Vr),t(R,bo),t(bo,Qr),t(R,Wr),t(R,go),t(go,Rr),d(e,Ea,l),p(Ye,e,l),d(e,Aa,l),d(e,K,l),t(K,pe),t(pe,vo),p(Je,vo,null),t(K,Kr),t(K,yo),t(yo,Xr),d(e,ka,l),d(e,B,l),t(B,Zr),t(B,wo),t(wo,en),t(B,tn),t(B,$o),t($o,on),t(B,an),t(B,Ve),t(Ve,Eo),t(Eo,sn),t(B,rn),d(e,Ca,l),d(e,X,l),t(X,_e),t(_e,Ao),p(Qe,Ao,null),t(X,nn),t(X,ko),t(ko,ln),d(e,za,l),d(e,w,l),p(We,w,null),t(w,dn),t(w,Re),t(Re,hn),t(Re,Co),t(Co,fn),t(Re,un),t(w,mn),t(w,Ke),t(Ke,cn),t(Ke,zo),t(zo,pn),t(Ke,_n),t(w,bn),t(w,M),t(M,gn),t(M,qo),t(qo,vn),t(M,yn),t(M,Po),t(Po,wn),t(M,$n),t(M,Bo),t(Bo,En),t(M,An),t(w,kn),t(w,be),p(Xe,be,null),t(be,Cn),t(be,Ze),t(Ze,zn),t(Ze,pt),t(pt,qn),t(Ze,Pn),t(w,Bn),t(w,ge),p(et,ge,null),t(ge,Tn),t(ge,To),t(To,Mn),t(w,Dn),t(w,ve),p(tt,ve,null),t(ve,xn),t(ve,ot),t(ot,Ln),t(ot,Mo),t(Mo,jn),t(ot,On),t(w,Sn),t(w,ye),p(at,ye,null),t(ye,Nn),t(ye,Do),t(Do,Fn),d(e,qa,l),d(e,Z,l),t(Z,we),t(we,xo),p(st,xo,null),t(Z,Hn),t(Z,_t),t(_t,Un),t(_t,Lo),t(Lo,In),d(e,Pa,l),d(e,S,l),t(S,Gn),t(S,rt),t(rt,Yn),t(S,Jn),t(S,jo),t(jo,Vn),t(S,Qn),Ba=!0},p(e,[l]){const nt={};l&2&&(nt.$$scope={dirty:l,ctx:e}),ne.$set(nt);const Oo={};l&2&&(Oo.$$scope={dirty:l,ctx:e}),le.$set(Oo)},i(e){Ba||(_(Ce.$$.fragment,e),_(ze.$$.fragment,e),_(Be.$$.fragment,e),_(Te.$$.fragment,e),_(Me.$$.fragment,e),_(ne.$$.fragment,e),_(De.$$.fragment,e),_(xe.$$.fragment,e),_(le.$$.fragment,e),_(Le.$$.fragment,e),_(je.$$.fragment,e),_(Oe.$$.fragment,e),_(Se.$$.fragment,e),_(Ne.$$.fragment,e),_(Fe.$$.fragment,e),_(He.$$.fragment,e),_(Ue.$$.fragment,e),_(Ie.$$.fragment,e),_(Ge.$$.fragment,e),_(Ye.$$.fragment,e),_(Je.$$.fragment,e),_(Qe.$$.fragment,e),_(We.$$.fragment,e),_(Xe.$$.fragment,e),_(et.$$.fragment,e),_(tt.$$.fragment,e),_(at.$$.fragment,e),_(st.$$.fragment,e),Ba=!0)},o(e){b(Ce.$$.fragment,e),b(ze.$$.fragment,e),b(Be.$$.fragment,e),b(Te.$$.fragment,e),b(Me.$$.fragment,e),b(ne.$$.fragment,e),b(De.$$.fragment,e),b(xe.$$.fragment,e),b(le.$$.fragment,e),b(Le.$$.fragment,e),b(je.$$.fragment,e),b(Oe.$$.fragment,e),b(Se.$$.fragment,e),b(Ne.$$.fragment,e),b(Fe.$$.fragment,e),b(He.$$.fragment,e),b(Ue.$$.fragment,e),b(Ie.$$.fragment,e),b(Ge.$$.fragment,e),b(Ye.$$.fragment,e),b(Je.$$.fragment,e),b(Qe.$$.fragment,e),b(We.$$.fragment,e),b(Xe.$$.fragment,e),b(et.$$.fragment,e),b(tt.$$.fragment,e),b(at.$$.fragment,e),b(st.$$.fragment,e),Ba=!1},d(e){o(v),e&&o(z),e&&o(y),g(Ce),e&&o(Ho),e&&o(U),g(ze),e&&o(Uo),e&&o(q),e&&o(Io),e&&o(D),e&&o(Go),e&&o(oe),e&&o(Yo),e&&o(I),g(Be),e&&o(Jo),e&&o(x),e&&o(Vo),g(Te,e),e&&o(Qo),e&&o(se),e&&o(Wo),e&&o(re),e&&o(Ro),g(Me,e),e&&o(Ko),e&&o(ht),e&&o(Xo),g(ne,e),e&&o(Zo),e&&o(G),g(De),e&&o(ea),e&&o(P),e&&o(ta),g(xe,e),e&&o(oa),g(le,e),e&&o(aa),e&&o(Y),g(Le),e&&o(sa),e&&o(L),e&&o(ra),g(je,e),e&&o(na),e&&o(A),e&&o(ia),e&&o(J),g(Oe),e&&o(la),e&&o(ft),e&&o(da),e&&o(V),g(Se),e&&o(ha),e&&o(k),e&&o(fa),e&&o(C),e&&o(ua),g(Ne,e),e&&o(ma),e&&o(j),e&&o(ca),g(Fe,e),e&&o(pa),e&&o(ut),e&&o(_a),g(He,e),e&&o(ba),e&&o(mt),e&&o(ga),e&&o(Q),g(Ue),e&&o(va),e&&o(O),e&&o(ya),g(Ie,e),e&&o(wa),e&&o(W),g(Ge),e&&o($a),e&&o(R),e&&o(Ea),g(Ye,e),e&&o(Aa),e&&o(K),g(Je),e&&o(ka),e&&o(B),e&&o(Ca),e&&o(X),g(Qe),e&&o(za),e&&o(w),g(We),g(Xe),g(et),g(tt),g(at),e&&o(qa),e&&o(Z),g(st),e&&o(Pa),e&&o(S)}}}const Hl={local:"quantize-transformers-models",sections:[{local:"bitsandbytes-integration",sections:[{local:"load-a-large-model-in-8bit",title:"Load a large model in 8bit"},{local:"push-quantized-models-on-the-hub",title:"Push quantized models on the \u{1F917} Hub"},{local:"load-a-quantized-model-from-the-hub",title:"Load a quantized model from the \u{1F917} Hub"},{local:"advanced-usecases",sections:[{local:"offload-between-cpu-and-gpu",title:"Offload between `cpu` and `gpu`"},{local:"play-with-llmint8threshold",title:"Play with `llm_int8_threshold`"},{local:"skip-the-conversion-of-some-modules",title:"Skip the conversion of some modules"},{local:"finetune-a-model-that-has-been-loaded-in-8bit",title:"Fine-tune a model that has been loaded in 8-bit"}],title:"Advanced usecases"},{local:"transformers.BitsAndBytesConfig",title:"BitsAndBytesConfig"}],title:"`bitsandbytes` Integration"},{local:"quantization-with-optimum",title:"Quantization with \u{1F917} `optimum` "}],title:"Quantize \u{1F917} Transformers models"};function Ul(it){return Ol(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ql extends Dl{constructor(v){super();xl(this,v,Ul,Fl,Ll,{})}}export{Ql as default,Hl as metadata};
