import{S as Ac,i as Sc,s as Nc,e as o,k as l,w as Gc,t as d,M as Yc,c as n,d as t,m as c,a,x as Lc,h as r,b as Ye,G as e,g as u,y as $c,L as Hc,q as Bc,o as Fc,B as Mc,v as Uc}from"../chunks/vendor-hf-doc-builder.js";import{I as Jc}from"../chunks/IconCopyLink-hf-doc-builder.js";function jc(Wr){let O,yo,C,W,Le,ce,hn,$e,fn,Oo,X,un,He,_n,pn,Co,m,mn,Be,gn,En,Fe,vn,Tn,Me,bn,Dn,ko,Z,xn,Ue,yn,On,wo,y,ee,Je,Cn,kn,je,wn,zn,qn,g,Ke,Rn,In,Qe,Pn,An,Ve,Sn,Nn,We,Gn,Yn,Ln,te,Xe,$n,Hn,Ze,Bn,Fn,zo,oe,Mn,et,Un,Jn,qo,E,p,tt,jn,Kn,ot,Qn,Vn,nt,Wn,Xn,at,Zn,ea,dt,ta,oa,na,v,rt,aa,da,lt,ra,la,ct,ca,ia,it,sa,ha,fa,T,st,ua,_a,ht,pa,ma,ft,ga,Ea,ut,va,Ta,ba,ne,_t,Da,xa,pt,ya,Oa,Ro,b,Ca,mt,ka,wa,gt,za,qa,Et,Ra,Ia,Io,_,Pa,vt,Aa,Sa,Tt,Na,Ga,bt,Ya,La,Dt,$a,Ha,xt,Ba,Fa,Po,ae,yt,k,Ot,Ma,Ua,Ct,Ja,ja,kt,Ka,Qa,s,w,wt,Va,Wa,zt,Xa,Za,qt,Rt,ed,td,z,Ao,od,It,nd,ad,ie,Pt,dd,rd,ld,q,So,cd,No,id,At,St,sd,hd,R,Go,fd,Nt,ud,_d,Gt,Yt,pd,md,I,Yo,gd,Lt,Ed,vd,$t,Ht,Td,bd,P,Lo,Dd,Bt,xd,yd,Ft,Od,Cd,A,Mt,kd,wd,Ut,zd,qd,se,Jt,Rd,Id,Pd,S,$o,Ad,Ho,Sd,jt,Kt,Nd,Gd,N,Bo,Yd,Qt,Ld,$d,he,Vt,Hd,Bd,Fd,G,Fo,Md,Mo,Ud,Wt,Xt,Jd,jd,Y,Uo,Kd,Zt,Qd,Vd,fe,eo,Wd,Xd,Zd,L,Jo,er,jo,tr,to,oo,or,nr,$,Ko,ar,no,dr,rr,ao,lr,cr,H,ro,ir,sr,lo,hr,fr,ue,co,ur,_r,pr,B,Qo,mr,Vo,gr,io,so,Er,vr,F,Wo,Tr,ho,br,Dr,_e,fo,xr,yr,Or,M,Xo,Cr,Zo,kr,uo,_o,wr,zr,U,en,qr,po,Rr,Ir,mo,Pr,Ar,J,tn,Sr,go,Nr,Gr,pe,Eo,Yr,Lr,$r,j,on,Hr,nn,Br,vo,To,Fr,an;return ce=new Jc({}),{c(){O=o("meta"),yo=l(),C=o("h1"),W=o("a"),Le=o("span"),Gc(ce.$$.fragment),hn=l(),$e=o("span"),fn=d("Padding and truncation"),Oo=l(),X=o("p"),un=d("Batched inputs are often different lengths, so they can\u2019t be converted to fixed-size tensors. Padding and truncation are strategies for dealing with this problem, to create rectangular tensors from batches of varying lengths. Padding adds a special "),He=o("strong"),_n=d("padding token"),pn=d(" to ensure shorter sequences will have the same length as either the longest sequence in a batch or the maximum length accepted by the model. Truncation works in the other direction by truncating long sequences."),Co=l(),m=o("p"),mn=d("In most cases, padding your batch to the length of the longest sequence and truncating to the maximum length a model can accept works pretty well. However, the API supports more strategies if you need them. The three arguments you need to are: "),Be=o("code"),gn=d("padding"),En=d(", "),Fe=o("code"),vn=d("truncation"),Tn=d(" and "),Me=o("code"),bn=d("max_length"),Dn=d("."),ko=l(),Z=o("p"),xn=d("The "),Ue=o("code"),yn=d("padding"),On=d(" argument controls padding. It can be a boolean or a string:"),wo=l(),y=o("ul"),ee=o("li"),Je=o("code"),Cn=d("True"),kn=d(" or "),je=o("code"),wn=d("'longest'"),zn=d(`: pad to the longest sequence in the batch (no padding is applied if you only provide
a single sequence).`),qn=l(),g=o("li"),Ke=o("code"),Rn=d("'max_length'"),In=d(": pad to a length specified by the "),Qe=o("code"),Pn=d("max_length"),An=d(` argument or the maximum length accepted
by the model if no `),Ve=o("code"),Sn=d("max_length"),Nn=d(" is provided ("),We=o("code"),Gn=d("max_length=None"),Yn=d("). Padding will still be applied if you only provide a single sequence."),Ln=l(),te=o("li"),Xe=o("code"),$n=d("False"),Hn=d(" or "),Ze=o("code"),Bn=d("'do_not_pad'"),Fn=d(": no padding is applied. This is the default behavior."),zo=l(),oe=o("p"),Mn=d("The "),et=o("code"),Un=d("truncation"),Jn=d(" argument controls truncation. It can be a boolean or a string:"),qo=l(),E=o("ul"),p=o("li"),tt=o("code"),jn=d("True"),Kn=d(" or "),ot=o("code"),Qn=d("'longest_first'"),Vn=d(": truncate to a maximum length specified by the "),nt=o("code"),Wn=d("max_length"),Xn=d(` argument or
the maximum length accepted by the model if no `),at=o("code"),Zn=d("max_length"),ea=d(" is provided ("),dt=o("code"),ta=d("max_length=None"),oa=d(`). This will
truncate token by token, removing a token from the longest sequence in the pair until the proper length is
reached.`),na=l(),v=o("li"),rt=o("code"),aa=d("'only_second'"),da=d(": truncate to a maximum length specified by the "),lt=o("code"),ra=d("max_length"),la=d(` argument or the maximum
length accepted by the model if no `),ct=o("code"),ca=d("max_length"),ia=d(" is provided ("),it=o("code"),sa=d("max_length=None"),ha=d(`). This will only truncate
the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.`),fa=l(),T=o("li"),st=o("code"),ua=d("'only_first'"),_a=d(": truncate to a maximum length specified by the "),ht=o("code"),pa=d("max_length"),ma=d(` argument or the maximum
length accepted by the model if no `),ft=o("code"),ga=d("max_length"),Ea=d(" is provided ("),ut=o("code"),va=d("max_length=None"),Ta=d(`). This will only truncate
the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.`),ba=l(),ne=o("li"),_t=o("code"),Da=d("False"),xa=d(" or "),pt=o("code"),ya=d("'do_not_truncate'"),Oa=d(": no truncation is applied. This is the default behavior."),Ro=l(),b=o("p"),Ca=d("The "),mt=o("code"),ka=d("max_length"),wa=d(" argument controls the length of the padding and truncation. It can be an integer or "),gt=o("code"),za=d("None"),qa=d(", in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to "),Et=o("code"),Ra=d("max_length"),Ia=d(" is deactivated."),Io=l(),_=o("p"),Pa=d("The following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace "),vt=o("code"),Aa=d("truncation=True"),Sa=d(" by a "),Tt=o("code"),Na=d("STRATEGY"),Ga=d(` selected in
`),bt=o("code"),Ya=d("['only_first', 'only_second', 'longest_first']"),La=d(", i.e. "),Dt=o("code"),$a=d("truncation='only_second'"),Ha=d(" or "),xt=o("code"),Ba=d("truncation='longest_first'"),Fa=d(" to control how both sequences in the pair are truncated as detailed before."),Po=l(),ae=o("table"),yt=o("thead"),k=o("tr"),Ot=o("th"),Ma=d("Truncation"),Ua=l(),Ct=o("th"),Ja=d("Padding"),ja=l(),kt=o("th"),Ka=d("Instruction"),Qa=l(),s=o("tbody"),w=o("tr"),wt=o("td"),Va=d("no truncation"),Wa=l(),zt=o("td"),Xa=d("no padding"),Za=l(),qt=o("td"),Rt=o("code"),ed=d("tokenizer(batch_sentences)"),td=l(),z=o("tr"),Ao=o("td"),od=l(),It=o("td"),nd=d("padding to max sequence in batch"),ad=l(),ie=o("td"),Pt=o("code"),dd=d("tokenizer(batch_sentences, padding=True)"),rd=d(" or"),ld=l(),q=o("tr"),So=o("td"),cd=l(),No=o("td"),id=l(),At=o("td"),St=o("code"),sd=d("tokenizer(batch_sentences, padding='longest')"),hd=l(),R=o("tr"),Go=o("td"),fd=l(),Nt=o("td"),ud=d("padding to max model input length"),_d=l(),Gt=o("td"),Yt=o("code"),pd=d("tokenizer(batch_sentences, padding='max_length')"),md=l(),I=o("tr"),Yo=o("td"),gd=l(),Lt=o("td"),Ed=d("padding to specific length"),vd=l(),$t=o("td"),Ht=o("code"),Td=d("tokenizer(batch_sentences, padding='max_length', max_length=42)"),bd=l(),P=o("tr"),Lo=o("td"),Dd=l(),Bt=o("td"),xd=d("padding to a multiple of a value"),yd=l(),Ft=o("td"),Od=d("`tokenizer(batch_sentences, padding=True, pad_to_multiple_of=8)"),Cd=l(),A=o("tr"),Mt=o("td"),kd=d("truncation to max model input length"),wd=l(),Ut=o("td"),zd=d("no padding"),qd=l(),se=o("td"),Jt=o("code"),Rd=d("tokenizer(batch_sentences, truncation=True)"),Id=d(" or"),Pd=l(),S=o("tr"),$o=o("td"),Ad=l(),Ho=o("td"),Sd=l(),jt=o("td"),Kt=o("code"),Nd=d("tokenizer(batch_sentences, truncation=STRATEGY)"),Gd=l(),N=o("tr"),Bo=o("td"),Yd=l(),Qt=o("td"),Ld=d("padding to max sequence in batch"),$d=l(),he=o("td"),Vt=o("code"),Hd=d("tokenizer(batch_sentences, padding=True, truncation=True)"),Bd=d(" or"),Fd=l(),G=o("tr"),Fo=o("td"),Md=l(),Mo=o("td"),Ud=l(),Wt=o("td"),Xt=o("code"),Jd=d("tokenizer(batch_sentences, padding=True, truncation=STRATEGY)"),jd=l(),Y=o("tr"),Uo=o("td"),Kd=l(),Zt=o("td"),Qd=d("padding to max model input length"),Vd=l(),fe=o("td"),eo=o("code"),Wd=d("tokenizer(batch_sentences, padding='max_length', truncation=True)"),Xd=d(" or"),Zd=l(),L=o("tr"),Jo=o("td"),er=l(),jo=o("td"),tr=l(),to=o("td"),oo=o("code"),or=d("tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY)"),nr=l(),$=o("tr"),Ko=o("td"),ar=l(),no=o("td"),dr=d("padding to specific length"),rr=l(),ao=o("td"),lr=d("Not possible"),cr=l(),H=o("tr"),ro=o("td"),ir=d("truncation to specific length"),sr=l(),lo=o("td"),hr=d("no padding"),fr=l(),ue=o("td"),co=o("code"),ur=d("tokenizer(batch_sentences, truncation=True, max_length=42)"),_r=d(" or"),pr=l(),B=o("tr"),Qo=o("td"),mr=l(),Vo=o("td"),gr=l(),io=o("td"),so=o("code"),Er=d("tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)"),vr=l(),F=o("tr"),Wo=o("td"),Tr=l(),ho=o("td"),br=d("padding to max sequence in batch"),Dr=l(),_e=o("td"),fo=o("code"),xr=d("tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)"),yr=d(" or"),Or=l(),M=o("tr"),Xo=o("td"),Cr=l(),Zo=o("td"),kr=l(),uo=o("td"),_o=o("code"),wr=d("tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)"),zr=l(),U=o("tr"),en=o("td"),qr=l(),po=o("td"),Rr=d("padding to max model input length"),Ir=l(),mo=o("td"),Pr=d("Not possible"),Ar=l(),J=o("tr"),tn=o("td"),Sr=l(),go=o("td"),Nr=d("padding to specific length"),Gr=l(),pe=o("td"),Eo=o("code"),Yr=d("tokenizer(batch_sentences, padding='max_length', truncation=True, max_length=42)"),Lr=d(" or"),$r=l(),j=o("tr"),on=o("td"),Hr=l(),nn=o("td"),Br=l(),vo=o("td"),To=o("code"),Fr=d("tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY, max_length=42)"),this.h()},l(i){const f=Yc('[data-svelte="svelte-1phssyn"]',document.head);O=n(f,"META",{name:!0,content:!0}),f.forEach(t),yo=c(i),C=n(i,"H1",{class:!0});var dn=a(C);W=n(dn,"A",{id:!0,class:!0,href:!0});var Xr=a(W);Le=n(Xr,"SPAN",{});var Zr=a(Le);Lc(ce.$$.fragment,Zr),Zr.forEach(t),Xr.forEach(t),hn=c(dn),$e=n(dn,"SPAN",{});var el=a($e);fn=r(el,"Padding and truncation"),el.forEach(t),dn.forEach(t),Oo=c(i),X=n(i,"P",{});var rn=a(X);un=r(rn,"Batched inputs are often different lengths, so they can\u2019t be converted to fixed-size tensors. Padding and truncation are strategies for dealing with this problem, to create rectangular tensors from batches of varying lengths. Padding adds a special "),He=n(rn,"STRONG",{});var tl=a(He);_n=r(tl,"padding token"),tl.forEach(t),pn=r(rn," to ensure shorter sequences will have the same length as either the longest sequence in a batch or the maximum length accepted by the model. Truncation works in the other direction by truncating long sequences."),rn.forEach(t),Co=c(i),m=n(i,"P",{});var de=a(m);mn=r(de,"In most cases, padding your batch to the length of the longest sequence and truncating to the maximum length a model can accept works pretty well. However, the API supports more strategies if you need them. The three arguments you need to are: "),Be=n(de,"CODE",{});var ol=a(Be);gn=r(ol,"padding"),ol.forEach(t),En=r(de,", "),Fe=n(de,"CODE",{});var nl=a(Fe);vn=r(nl,"truncation"),nl.forEach(t),Tn=r(de," and "),Me=n(de,"CODE",{});var al=a(Me);bn=r(al,"max_length"),al.forEach(t),Dn=r(de,"."),de.forEach(t),ko=c(i),Z=n(i,"P",{});var ln=a(Z);xn=r(ln,"The "),Ue=n(ln,"CODE",{});var dl=a(Ue);yn=r(dl,"padding"),dl.forEach(t),On=r(ln," argument controls padding. It can be a boolean or a string:"),ln.forEach(t),wo=c(i),y=n(i,"UL",{});var me=a(y);ee=n(me,"LI",{});var bo=a(ee);Je=n(bo,"CODE",{});var rl=a(Je);Cn=r(rl,"True"),rl.forEach(t),kn=r(bo," or "),je=n(bo,"CODE",{});var ll=a(je);wn=r(ll,"'longest'"),ll.forEach(t),zn=r(bo,`: pad to the longest sequence in the batch (no padding is applied if you only provide
a single sequence).`),bo.forEach(t),qn=c(me),g=n(me,"LI",{});var K=a(g);Ke=n(K,"CODE",{});var cl=a(Ke);Rn=r(cl,"'max_length'"),cl.forEach(t),In=r(K,": pad to a length specified by the "),Qe=n(K,"CODE",{});var il=a(Qe);Pn=r(il,"max_length"),il.forEach(t),An=r(K,` argument or the maximum length accepted
by the model if no `),Ve=n(K,"CODE",{});var sl=a(Ve);Sn=r(sl,"max_length"),sl.forEach(t),Nn=r(K," is provided ("),We=n(K,"CODE",{});var hl=a(We);Gn=r(hl,"max_length=None"),hl.forEach(t),Yn=r(K,"). Padding will still be applied if you only provide a single sequence."),K.forEach(t),Ln=c(me),te=n(me,"LI",{});var Do=a(te);Xe=n(Do,"CODE",{});var fl=a(Xe);$n=r(fl,"False"),fl.forEach(t),Hn=r(Do," or "),Ze=n(Do,"CODE",{});var ul=a(Ze);Bn=r(ul,"'do_not_pad'"),ul.forEach(t),Fn=r(Do,": no padding is applied. This is the default behavior."),Do.forEach(t),me.forEach(t),zo=c(i),oe=n(i,"P",{});var cn=a(oe);Mn=r(cn,"The "),et=n(cn,"CODE",{});var _l=a(et);Un=r(_l,"truncation"),_l.forEach(t),Jn=r(cn," argument controls truncation. It can be a boolean or a string:"),cn.forEach(t),qo=c(i),E=n(i,"UL",{});var re=a(E);p=n(re,"LI",{});var x=a(p);tt=n(x,"CODE",{});var pl=a(tt);jn=r(pl,"True"),pl.forEach(t),Kn=r(x," or "),ot=n(x,"CODE",{});var ml=a(ot);Qn=r(ml,"'longest_first'"),ml.forEach(t),Vn=r(x,": truncate to a maximum length specified by the "),nt=n(x,"CODE",{});var gl=a(nt);Wn=r(gl,"max_length"),gl.forEach(t),Xn=r(x,` argument or
the maximum length accepted by the model if no `),at=n(x,"CODE",{});var El=a(at);Zn=r(El,"max_length"),El.forEach(t),ea=r(x," is provided ("),dt=n(x,"CODE",{});var vl=a(dt);ta=r(vl,"max_length=None"),vl.forEach(t),oa=r(x,`). This will
truncate token by token, removing a token from the longest sequence in the pair until the proper length is
reached.`),x.forEach(t),na=c(re),v=n(re,"LI",{});var Q=a(v);rt=n(Q,"CODE",{});var Tl=a(rt);aa=r(Tl,"'only_second'"),Tl.forEach(t),da=r(Q,": truncate to a maximum length specified by the "),lt=n(Q,"CODE",{});var bl=a(lt);ra=r(bl,"max_length"),bl.forEach(t),la=r(Q,` argument or the maximum
length accepted by the model if no `),ct=n(Q,"CODE",{});var Dl=a(ct);ca=r(Dl,"max_length"),Dl.forEach(t),ia=r(Q," is provided ("),it=n(Q,"CODE",{});var xl=a(it);sa=r(xl,"max_length=None"),xl.forEach(t),ha=r(Q,`). This will only truncate
the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.`),Q.forEach(t),fa=c(re),T=n(re,"LI",{});var V=a(T);st=n(V,"CODE",{});var yl=a(st);ua=r(yl,"'only_first'"),yl.forEach(t),_a=r(V,": truncate to a maximum length specified by the "),ht=n(V,"CODE",{});var Ol=a(ht);pa=r(Ol,"max_length"),Ol.forEach(t),ma=r(V,` argument or the maximum
length accepted by the model if no `),ft=n(V,"CODE",{});var Cl=a(ft);ga=r(Cl,"max_length"),Cl.forEach(t),Ea=r(V," is provided ("),ut=n(V,"CODE",{});var kl=a(ut);va=r(kl,"max_length=None"),kl.forEach(t),Ta=r(V,`). This will only truncate
the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.`),V.forEach(t),ba=c(re),ne=n(re,"LI",{});var xo=a(ne);_t=n(xo,"CODE",{});var wl=a(_t);Da=r(wl,"False"),wl.forEach(t),xa=r(xo," or "),pt=n(xo,"CODE",{});var zl=a(pt);ya=r(zl,"'do_not_truncate'"),zl.forEach(t),Oa=r(xo,": no truncation is applied. This is the default behavior."),xo.forEach(t),re.forEach(t),Ro=c(i),b=n(i,"P",{});var le=a(b);Ca=r(le,"The "),mt=n(le,"CODE",{});var ql=a(mt);ka=r(ql,"max_length"),ql.forEach(t),wa=r(le," argument controls the length of the padding and truncation. It can be an integer or "),gt=n(le,"CODE",{});var Rl=a(gt);za=r(Rl,"None"),Rl.forEach(t),qa=r(le,", in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to "),Et=n(le,"CODE",{});var Il=a(Et);Ra=r(Il,"max_length"),Il.forEach(t),Ia=r(le," is deactivated."),le.forEach(t),Io=c(i),_=n(i,"P",{});var D=a(_);Pa=r(D,"The following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace "),vt=n(D,"CODE",{});var Pl=a(vt);Aa=r(Pl,"truncation=True"),Pl.forEach(t),Sa=r(D," by a "),Tt=n(D,"CODE",{});var Al=a(Tt);Na=r(Al,"STRATEGY"),Al.forEach(t),Ga=r(D,` selected in
`),bt=n(D,"CODE",{});var Sl=a(bt);Ya=r(Sl,"['only_first', 'only_second', 'longest_first']"),Sl.forEach(t),La=r(D,", i.e. "),Dt=n(D,"CODE",{});var Nl=a(Dt);$a=r(Nl,"truncation='only_second'"),Nl.forEach(t),Ha=r(D," or "),xt=n(D,"CODE",{});var Gl=a(xt);Ba=r(Gl,"truncation='longest_first'"),Gl.forEach(t),Fa=r(D," to control how both sequences in the pair are truncated as detailed before."),D.forEach(t),Po=c(i),ae=n(i,"TABLE",{});var sn=a(ae);yt=n(sn,"THEAD",{});var Yl=a(yt);k=n(Yl,"TR",{});var ge=a(k);Ot=n(ge,"TH",{});var Ll=a(Ot);Ma=r(Ll,"Truncation"),Ll.forEach(t),Ua=c(ge),Ct=n(ge,"TH",{});var $l=a(Ct);Ja=r($l,"Padding"),$l.forEach(t),ja=c(ge),kt=n(ge,"TH",{});var Hl=a(kt);Ka=r(Hl,"Instruction"),Hl.forEach(t),ge.forEach(t),Yl.forEach(t),Qa=c(sn),s=n(sn,"TBODY",{});var h=a(s);w=n(h,"TR",{});var Ee=a(w);wt=n(Ee,"TD",{});var Bl=a(wt);Va=r(Bl,"no truncation"),Bl.forEach(t),Wa=c(Ee),zt=n(Ee,"TD",{});var Fl=a(zt);Xa=r(Fl,"no padding"),Fl.forEach(t),Za=c(Ee),qt=n(Ee,"TD",{});var Ml=a(qt);Rt=n(Ml,"CODE",{});var Ul=a(Rt);ed=r(Ul,"tokenizer(batch_sentences)"),Ul.forEach(t),Ml.forEach(t),Ee.forEach(t),td=c(h),z=n(h,"TR",{});var ve=a(z);Ao=n(ve,"TD",{}),a(Ao).forEach(t),od=c(ve),It=n(ve,"TD",{});var Jl=a(It);nd=r(Jl,"padding to max sequence in batch"),Jl.forEach(t),ad=c(ve),ie=n(ve,"TD",{});var Mr=a(ie);Pt=n(Mr,"CODE",{});var jl=a(Pt);dd=r(jl,"tokenizer(batch_sentences, padding=True)"),jl.forEach(t),rd=r(Mr," or"),Mr.forEach(t),ve.forEach(t),ld=c(h),q=n(h,"TR",{});var Te=a(q);So=n(Te,"TD",{}),a(So).forEach(t),cd=c(Te),No=n(Te,"TD",{}),a(No).forEach(t),id=c(Te),At=n(Te,"TD",{});var Kl=a(At);St=n(Kl,"CODE",{});var Ql=a(St);sd=r(Ql,"tokenizer(batch_sentences, padding='longest')"),Ql.forEach(t),Kl.forEach(t),Te.forEach(t),hd=c(h),R=n(h,"TR",{});var be=a(R);Go=n(be,"TD",{}),a(Go).forEach(t),fd=c(be),Nt=n(be,"TD",{});var Vl=a(Nt);ud=r(Vl,"padding to max model input length"),Vl.forEach(t),_d=c(be),Gt=n(be,"TD",{});var Wl=a(Gt);Yt=n(Wl,"CODE",{});var Xl=a(Yt);pd=r(Xl,"tokenizer(batch_sentences, padding='max_length')"),Xl.forEach(t),Wl.forEach(t),be.forEach(t),md=c(h),I=n(h,"TR",{});var De=a(I);Yo=n(De,"TD",{}),a(Yo).forEach(t),gd=c(De),Lt=n(De,"TD",{});var Zl=a(Lt);Ed=r(Zl,"padding to specific length"),Zl.forEach(t),vd=c(De),$t=n(De,"TD",{});var ec=a($t);Ht=n(ec,"CODE",{});var tc=a(Ht);Td=r(tc,"tokenizer(batch_sentences, padding='max_length', max_length=42)"),tc.forEach(t),ec.forEach(t),De.forEach(t),bd=c(h),P=n(h,"TR",{});var xe=a(P);Lo=n(xe,"TD",{}),a(Lo).forEach(t),Dd=c(xe),Bt=n(xe,"TD",{});var oc=a(Bt);xd=r(oc,"padding to a multiple of a value"),oc.forEach(t),yd=c(xe),Ft=n(xe,"TD",{});var nc=a(Ft);Od=r(nc,"`tokenizer(batch_sentences, padding=True, pad_to_multiple_of=8)"),nc.forEach(t),xe.forEach(t),Cd=c(h),A=n(h,"TR",{});var ye=a(A);Mt=n(ye,"TD",{});var ac=a(Mt);kd=r(ac,"truncation to max model input length"),ac.forEach(t),wd=c(ye),Ut=n(ye,"TD",{});var dc=a(Ut);zd=r(dc,"no padding"),dc.forEach(t),qd=c(ye),se=n(ye,"TD",{});var Ur=a(se);Jt=n(Ur,"CODE",{});var rc=a(Jt);Rd=r(rc,"tokenizer(batch_sentences, truncation=True)"),rc.forEach(t),Id=r(Ur," or"),Ur.forEach(t),ye.forEach(t),Pd=c(h),S=n(h,"TR",{});var Oe=a(S);$o=n(Oe,"TD",{}),a($o).forEach(t),Ad=c(Oe),Ho=n(Oe,"TD",{}),a(Ho).forEach(t),Sd=c(Oe),jt=n(Oe,"TD",{});var lc=a(jt);Kt=n(lc,"CODE",{});var cc=a(Kt);Nd=r(cc,"tokenizer(batch_sentences, truncation=STRATEGY)"),cc.forEach(t),lc.forEach(t),Oe.forEach(t),Gd=c(h),N=n(h,"TR",{});var Ce=a(N);Bo=n(Ce,"TD",{}),a(Bo).forEach(t),Yd=c(Ce),Qt=n(Ce,"TD",{});var ic=a(Qt);Ld=r(ic,"padding to max sequence in batch"),ic.forEach(t),$d=c(Ce),he=n(Ce,"TD",{});var Jr=a(he);Vt=n(Jr,"CODE",{});var sc=a(Vt);Hd=r(sc,"tokenizer(batch_sentences, padding=True, truncation=True)"),sc.forEach(t),Bd=r(Jr," or"),Jr.forEach(t),Ce.forEach(t),Fd=c(h),G=n(h,"TR",{});var ke=a(G);Fo=n(ke,"TD",{}),a(Fo).forEach(t),Md=c(ke),Mo=n(ke,"TD",{}),a(Mo).forEach(t),Ud=c(ke),Wt=n(ke,"TD",{});var hc=a(Wt);Xt=n(hc,"CODE",{});var fc=a(Xt);Jd=r(fc,"tokenizer(batch_sentences, padding=True, truncation=STRATEGY)"),fc.forEach(t),hc.forEach(t),ke.forEach(t),jd=c(h),Y=n(h,"TR",{});var we=a(Y);Uo=n(we,"TD",{}),a(Uo).forEach(t),Kd=c(we),Zt=n(we,"TD",{});var uc=a(Zt);Qd=r(uc,"padding to max model input length"),uc.forEach(t),Vd=c(we),fe=n(we,"TD",{});var jr=a(fe);eo=n(jr,"CODE",{});var _c=a(eo);Wd=r(_c,"tokenizer(batch_sentences, padding='max_length', truncation=True)"),_c.forEach(t),Xd=r(jr," or"),jr.forEach(t),we.forEach(t),Zd=c(h),L=n(h,"TR",{});var ze=a(L);Jo=n(ze,"TD",{}),a(Jo).forEach(t),er=c(ze),jo=n(ze,"TD",{}),a(jo).forEach(t),tr=c(ze),to=n(ze,"TD",{});var pc=a(to);oo=n(pc,"CODE",{});var mc=a(oo);or=r(mc,"tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY)"),mc.forEach(t),pc.forEach(t),ze.forEach(t),nr=c(h),$=n(h,"TR",{});var qe=a($);Ko=n(qe,"TD",{}),a(Ko).forEach(t),ar=c(qe),no=n(qe,"TD",{});var gc=a(no);dr=r(gc,"padding to specific length"),gc.forEach(t),rr=c(qe),ao=n(qe,"TD",{});var Ec=a(ao);lr=r(Ec,"Not possible"),Ec.forEach(t),qe.forEach(t),cr=c(h),H=n(h,"TR",{});var Re=a(H);ro=n(Re,"TD",{});var vc=a(ro);ir=r(vc,"truncation to specific length"),vc.forEach(t),sr=c(Re),lo=n(Re,"TD",{});var Tc=a(lo);hr=r(Tc,"no padding"),Tc.forEach(t),fr=c(Re),ue=n(Re,"TD",{});var Kr=a(ue);co=n(Kr,"CODE",{});var bc=a(co);ur=r(bc,"tokenizer(batch_sentences, truncation=True, max_length=42)"),bc.forEach(t),_r=r(Kr," or"),Kr.forEach(t),Re.forEach(t),pr=c(h),B=n(h,"TR",{});var Ie=a(B);Qo=n(Ie,"TD",{}),a(Qo).forEach(t),mr=c(Ie),Vo=n(Ie,"TD",{}),a(Vo).forEach(t),gr=c(Ie),io=n(Ie,"TD",{});var Dc=a(io);so=n(Dc,"CODE",{});var xc=a(so);Er=r(xc,"tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)"),xc.forEach(t),Dc.forEach(t),Ie.forEach(t),vr=c(h),F=n(h,"TR",{});var Pe=a(F);Wo=n(Pe,"TD",{}),a(Wo).forEach(t),Tr=c(Pe),ho=n(Pe,"TD",{});var yc=a(ho);br=r(yc,"padding to max sequence in batch"),yc.forEach(t),Dr=c(Pe),_e=n(Pe,"TD",{});var Qr=a(_e);fo=n(Qr,"CODE",{});var Oc=a(fo);xr=r(Oc,"tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)"),Oc.forEach(t),yr=r(Qr," or"),Qr.forEach(t),Pe.forEach(t),Or=c(h),M=n(h,"TR",{});var Ae=a(M);Xo=n(Ae,"TD",{}),a(Xo).forEach(t),Cr=c(Ae),Zo=n(Ae,"TD",{}),a(Zo).forEach(t),kr=c(Ae),uo=n(Ae,"TD",{});var Cc=a(uo);_o=n(Cc,"CODE",{});var kc=a(_o);wr=r(kc,"tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)"),kc.forEach(t),Cc.forEach(t),Ae.forEach(t),zr=c(h),U=n(h,"TR",{});var Se=a(U);en=n(Se,"TD",{}),a(en).forEach(t),qr=c(Se),po=n(Se,"TD",{});var wc=a(po);Rr=r(wc,"padding to max model input length"),wc.forEach(t),Ir=c(Se),mo=n(Se,"TD",{});var zc=a(mo);Pr=r(zc,"Not possible"),zc.forEach(t),Se.forEach(t),Ar=c(h),J=n(h,"TR",{});var Ne=a(J);tn=n(Ne,"TD",{}),a(tn).forEach(t),Sr=c(Ne),go=n(Ne,"TD",{});var qc=a(go);Nr=r(qc,"padding to specific length"),qc.forEach(t),Gr=c(Ne),pe=n(Ne,"TD",{});var Vr=a(pe);Eo=n(Vr,"CODE",{});var Rc=a(Eo);Yr=r(Rc,"tokenizer(batch_sentences, padding='max_length', truncation=True, max_length=42)"),Rc.forEach(t),Lr=r(Vr," or"),Vr.forEach(t),Ne.forEach(t),$r=c(h),j=n(h,"TR",{});var Ge=a(j);on=n(Ge,"TD",{}),a(on).forEach(t),Hr=c(Ge),nn=n(Ge,"TD",{}),a(nn).forEach(t),Br=c(Ge),vo=n(Ge,"TD",{});var Ic=a(vo);To=n(Ic,"CODE",{});var Pc=a(To);Fr=r(Pc,"tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY, max_length=42)"),Pc.forEach(t),Ic.forEach(t),Ge.forEach(t),h.forEach(t),sn.forEach(t),this.h()},h(){Ye(O,"name","hf:doc:metadata"),Ye(O,"content",JSON.stringify(Kc)),Ye(W,"id","padding-and-truncation"),Ye(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),Ye(W,"href","#padding-and-truncation"),Ye(C,"class","relative group")},m(i,f){e(document.head,O),u(i,yo,f),u(i,C,f),e(C,W),e(W,Le),$c(ce,Le,null),e(C,hn),e(C,$e),e($e,fn),u(i,Oo,f),u(i,X,f),e(X,un),e(X,He),e(He,_n),e(X,pn),u(i,Co,f),u(i,m,f),e(m,mn),e(m,Be),e(Be,gn),e(m,En),e(m,Fe),e(Fe,vn),e(m,Tn),e(m,Me),e(Me,bn),e(m,Dn),u(i,ko,f),u(i,Z,f),e(Z,xn),e(Z,Ue),e(Ue,yn),e(Z,On),u(i,wo,f),u(i,y,f),e(y,ee),e(ee,Je),e(Je,Cn),e(ee,kn),e(ee,je),e(je,wn),e(ee,zn),e(y,qn),e(y,g),e(g,Ke),e(Ke,Rn),e(g,In),e(g,Qe),e(Qe,Pn),e(g,An),e(g,Ve),e(Ve,Sn),e(g,Nn),e(g,We),e(We,Gn),e(g,Yn),e(y,Ln),e(y,te),e(te,Xe),e(Xe,$n),e(te,Hn),e(te,Ze),e(Ze,Bn),e(te,Fn),u(i,zo,f),u(i,oe,f),e(oe,Mn),e(oe,et),e(et,Un),e(oe,Jn),u(i,qo,f),u(i,E,f),e(E,p),e(p,tt),e(tt,jn),e(p,Kn),e(p,ot),e(ot,Qn),e(p,Vn),e(p,nt),e(nt,Wn),e(p,Xn),e(p,at),e(at,Zn),e(p,ea),e(p,dt),e(dt,ta),e(p,oa),e(E,na),e(E,v),e(v,rt),e(rt,aa),e(v,da),e(v,lt),e(lt,ra),e(v,la),e(v,ct),e(ct,ca),e(v,ia),e(v,it),e(it,sa),e(v,ha),e(E,fa),e(E,T),e(T,st),e(st,ua),e(T,_a),e(T,ht),e(ht,pa),e(T,ma),e(T,ft),e(ft,ga),e(T,Ea),e(T,ut),e(ut,va),e(T,Ta),e(E,ba),e(E,ne),e(ne,_t),e(_t,Da),e(ne,xa),e(ne,pt),e(pt,ya),e(ne,Oa),u(i,Ro,f),u(i,b,f),e(b,Ca),e(b,mt),e(mt,ka),e(b,wa),e(b,gt),e(gt,za),e(b,qa),e(b,Et),e(Et,Ra),e(b,Ia),u(i,Io,f),u(i,_,f),e(_,Pa),e(_,vt),e(vt,Aa),e(_,Sa),e(_,Tt),e(Tt,Na),e(_,Ga),e(_,bt),e(bt,Ya),e(_,La),e(_,Dt),e(Dt,$a),e(_,Ha),e(_,xt),e(xt,Ba),e(_,Fa),u(i,Po,f),u(i,ae,f),e(ae,yt),e(yt,k),e(k,Ot),e(Ot,Ma),e(k,Ua),e(k,Ct),e(Ct,Ja),e(k,ja),e(k,kt),e(kt,Ka),e(ae,Qa),e(ae,s),e(s,w),e(w,wt),e(wt,Va),e(w,Wa),e(w,zt),e(zt,Xa),e(w,Za),e(w,qt),e(qt,Rt),e(Rt,ed),e(s,td),e(s,z),e(z,Ao),e(z,od),e(z,It),e(It,nd),e(z,ad),e(z,ie),e(ie,Pt),e(Pt,dd),e(ie,rd),e(s,ld),e(s,q),e(q,So),e(q,cd),e(q,No),e(q,id),e(q,At),e(At,St),e(St,sd),e(s,hd),e(s,R),e(R,Go),e(R,fd),e(R,Nt),e(Nt,ud),e(R,_d),e(R,Gt),e(Gt,Yt),e(Yt,pd),e(s,md),e(s,I),e(I,Yo),e(I,gd),e(I,Lt),e(Lt,Ed),e(I,vd),e(I,$t),e($t,Ht),e(Ht,Td),e(s,bd),e(s,P),e(P,Lo),e(P,Dd),e(P,Bt),e(Bt,xd),e(P,yd),e(P,Ft),e(Ft,Od),e(s,Cd),e(s,A),e(A,Mt),e(Mt,kd),e(A,wd),e(A,Ut),e(Ut,zd),e(A,qd),e(A,se),e(se,Jt),e(Jt,Rd),e(se,Id),e(s,Pd),e(s,S),e(S,$o),e(S,Ad),e(S,Ho),e(S,Sd),e(S,jt),e(jt,Kt),e(Kt,Nd),e(s,Gd),e(s,N),e(N,Bo),e(N,Yd),e(N,Qt),e(Qt,Ld),e(N,$d),e(N,he),e(he,Vt),e(Vt,Hd),e(he,Bd),e(s,Fd),e(s,G),e(G,Fo),e(G,Md),e(G,Mo),e(G,Ud),e(G,Wt),e(Wt,Xt),e(Xt,Jd),e(s,jd),e(s,Y),e(Y,Uo),e(Y,Kd),e(Y,Zt),e(Zt,Qd),e(Y,Vd),e(Y,fe),e(fe,eo),e(eo,Wd),e(fe,Xd),e(s,Zd),e(s,L),e(L,Jo),e(L,er),e(L,jo),e(L,tr),e(L,to),e(to,oo),e(oo,or),e(s,nr),e(s,$),e($,Ko),e($,ar),e($,no),e(no,dr),e($,rr),e($,ao),e(ao,lr),e(s,cr),e(s,H),e(H,ro),e(ro,ir),e(H,sr),e(H,lo),e(lo,hr),e(H,fr),e(H,ue),e(ue,co),e(co,ur),e(ue,_r),e(s,pr),e(s,B),e(B,Qo),e(B,mr),e(B,Vo),e(B,gr),e(B,io),e(io,so),e(so,Er),e(s,vr),e(s,F),e(F,Wo),e(F,Tr),e(F,ho),e(ho,br),e(F,Dr),e(F,_e),e(_e,fo),e(fo,xr),e(_e,yr),e(s,Or),e(s,M),e(M,Xo),e(M,Cr),e(M,Zo),e(M,kr),e(M,uo),e(uo,_o),e(_o,wr),e(s,zr),e(s,U),e(U,en),e(U,qr),e(U,po),e(po,Rr),e(U,Ir),e(U,mo),e(mo,Pr),e(s,Ar),e(s,J),e(J,tn),e(J,Sr),e(J,go),e(go,Nr),e(J,Gr),e(J,pe),e(pe,Eo),e(Eo,Yr),e(pe,Lr),e(s,$r),e(s,j),e(j,on),e(j,Hr),e(j,nn),e(j,Br),e(j,vo),e(vo,To),e(To,Fr),an=!0},p:Hc,i(i){an||(Bc(ce.$$.fragment,i),an=!0)},o(i){Fc(ce.$$.fragment,i),an=!1},d(i){t(O),i&&t(yo),i&&t(C),Mc(ce),i&&t(Oo),i&&t(X),i&&t(Co),i&&t(m),i&&t(ko),i&&t(Z),i&&t(wo),i&&t(y),i&&t(zo),i&&t(oe),i&&t(qo),i&&t(E),i&&t(Ro),i&&t(b),i&&t(Io),i&&t(_),i&&t(Po),i&&t(ae)}}}const Kc={local:"padding-and-truncation",title:"Padding and truncation"};function Qc(Wr){return Uc(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Xc extends Ac{constructor(O){super();Sc(this,O,Qc,jc,Nc,{})}}export{Xc as default,Kc as metadata};
