import{S as na,i as aa,s as ta,e as a,k as i,w as b,t as r,M as la,c as t,d as n,m as c,a as l,x as y,h as o,b as d,G as e,g as m,y as f,L as ra,q as g,o as j,B as T,v as oa}from"../../chunks/vendor-hf-doc-builder.js";import{D as Es}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Mn}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as he}from"../../chunks/IconCopyLink-hf-doc-builder.js";function pa(kn){let _,Ws,U,I,fs,N,ue,gs,be,Cs,J,G,js,q,ye,Ts,fe,Rs,z,ge,$,je,Te,Vs,as,Me,Ns,ts,Ms,ke,qs,M,we,X,_e,Ue,Q,Je,ve,$s,x,Be,ls,Ie,Ge,Xs,rs,ze,Qs,v,Z,ks,F,xe,ws,Ze,Fs,os,Ee,Ss,S,As,ps,We,Hs,A,Ls,k,Ce,is,Re,Ve,_s,Ne,qe,Ys,H,Ds,B,E,Us,L,$e,Js,Xe,Ps,h,Y,Qe,vs,Fe,Se,D,Ae,cs,He,Le,Ye,w,P,De,Bs,Pe,Oe,O,ds,Ke,Is,sn,en,ms,nn,Gs,an,tn,W,K,ln,zs,rn,on,C,ss,pn,xs,cn,dn,R,es,mn,ns,hn,Zs,un,bn,Os,V,yn,hs,fn,gn,Ks;return N=new he({}),q=new he({}),F=new he({}),S=new Mn({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBaW1wb3J0JTIwdG9yY2glMEElMEFtb2RlbCUyMCUzRCUyMFQ1Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUlMkZieXQ1LXNtYWxsJTIyKSUwQSUwQW51bV9zcGVjaWFsX3Rva2VucyUyMCUzRCUyMDMlMEElMjMlMjBNb2RlbCUyMGhhcyUyMDMlMjBzcGVjaWFsJTIwdG9rZW5zJTIwd2hpY2glMjB0YWtlJTIwdXAlMjB0aGUlMjBpbnB1dCUyMGlkcyUyMDAlMkMxJTJDMiUyMG9mJTIwQnlUNS4lMEElMjMlMjAlM0QlM0UlMjBOZWVkJTIwdG8lMjBzaGlmdCUyMHV0Zi04JTIwY2hhcmFjdGVyJTIwZW5jb2RpbmdzJTIwYnklMjAzJTIwYmVmb3JlJTIwcGFzc2luZyUyMGlkcyUyMHRvJTIwbW9kZWwuJTBBJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9yY2gudGVuc29yKCU1Qmxpc3QoJTIyTGlmZSUyMGlzJTIwbGlrZSUyMGElMjBib3glMjBvZiUyMGNob2NvbGF0ZXMuJTIyLmVuY29kZSglMjJ1dGYtOCUyMikpJTVEKSUyMCUyQiUyMG51bV9zcGVjaWFsX3Rva2VucyUwQSUwQWxhYmVscyUyMCUzRCUyMHRvcmNoLnRlbnNvciglNUJsaXN0KCUyMkxhJTIwdmllJTIwZXN0JTIwY29tbWUlMjB1bmUlMjBibyVDMyVBRXRlJTIwZGUlMjBjaG9jb2xhdC4lMjIuZW5jb2RlKCUyMnV0Zi04JTIyKSklNUQpJTIwJTJCJTIwbnVtX3NwZWNpYWxfdG9rZW5zJTBBJTBBbG9zcyUyMCUzRCUyMG1vZGVsKGlucHV0X2lkcyUyQyUyMGxhYmVscyUzRGxhYmVscykubG9zcyUwQWxvc3MuaXRlbSgp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5ForConditionalGeneration
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>model = T5ForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/byt5-small&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>num_special_tokens = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Model has 3 special tokens which take up the input ids 0,1,2 of ByT5.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># =&gt; Need to shift utf-8 character encodings by 3 before passing ids to model.</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = torch.tensor([<span class="hljs-built_in">list</span>(<span class="hljs-string">&quot;Life is like a box of chocolates.&quot;</span>.encode(<span class="hljs-string">&quot;utf-8&quot;</span>))]) + num_special_tokens

<span class="hljs-meta">&gt;&gt;&gt; </span>labels = torch.tensor([<span class="hljs-built_in">list</span>(<span class="hljs-string">&quot;La vie est comme une bo\xEEte de chocolat.&quot;</span>.encode(<span class="hljs-string">&quot;utf-8&quot;</span>))]) + num_special_tokens

<span class="hljs-meta">&gt;&gt;&gt; </span>loss = model(input_ids, labels=labels).loss
<span class="hljs-meta">&gt;&gt;&gt; </span>loss.item()
<span class="hljs-number">2.66</span>`}}),A=new Mn({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwVDVGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmJ5dDUtc21hbGwlMjIpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGYnl0NS1zbWFsbCUyMiklMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTBBJTIwJTIwJTIwJTIwJTVCJTIyTGlmZSUyMGlzJTIwbGlrZSUyMGElMjBib3glMjBvZiUyMGNob2NvbGF0ZXMuJTIyJTJDJTIwJTIyVG9kYXklMjBpcyUyME1vbmRheS4lMjIlNUQlMkMlMjBwYWRkaW5nJTNEJTIybG9uZ2VzdCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMEEpJTBBbGFiZWxzX2RpY3QlMjAlM0QlMjB0b2tlbml6ZXIoJTBBJTIwJTIwJTIwJTIwJTVCJTIyTGElMjB2aWUlMjBlc3QlMjBjb21tZSUyMHVuZSUyMGJvJUMzJUFFdGUlMjBkZSUyMGNob2NvbGF0LiUyMiUyQyUyMCUyMkF1am91cmQnaHVpJTIwYydlc3QlMjBsdW5kaS4lMjIlNUQlMkMlMjBwYWRkaW5nJTNEJTIybG9uZ2VzdCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMEEpJTBBbGFiZWxzJTIwJTNEJTIwbGFiZWxzX2RpY3QuaW5wdXRfaWRzJTBBJTBBbG9zcyUyMCUzRCUyMG1vZGVsKCoqbW9kZWxfaW5wdXRzJTJDJTIwbGFiZWxzJTNEbGFiZWxzKS5sb3NzJTBBbG9zcy5pdGVtKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5ForConditionalGeneration, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = T5ForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/byt5-small&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google/byt5-small&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model_inputs = tokenizer(
<span class="hljs-meta">... </span>    [<span class="hljs-string">&quot;Life is like a box of chocolates.&quot;</span>, <span class="hljs-string">&quot;Today is Monday.&quot;</span>], padding=<span class="hljs-string">&quot;longest&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>labels_dict = tokenizer(
<span class="hljs-meta">... </span>    [<span class="hljs-string">&quot;La vie est comme une bo\xEEte de chocolat.&quot;</span>, <span class="hljs-string">&quot;Aujourd&#x27;hui c&#x27;est lundi.&quot;</span>], padding=<span class="hljs-string">&quot;longest&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>labels = labels_dict.input_ids

<span class="hljs-meta">&gt;&gt;&gt; </span>loss = model(**model_inputs, labels=labels).loss
<span class="hljs-meta">&gt;&gt;&gt; </span>loss.item()
<span class="hljs-number">17.9</span>`}}),H=new Mn({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JTZXEyU2VxTE0lMEFpbXBvcnQlMjB0b3JjaCUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmJ5dDUtYmFzZSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTS5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGYnl0NS1iYXNlJTIyKSUwQSUwQWlucHV0X2lkc19wcm9tcHQlMjAlM0QlMjAlMjJUaGUlMjBkb2clMjBjaGFzZXMlMjBhJTIwYmFsbCUyMGluJTIwdGhlJTIwcGFyay4lMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfaWRzX3Byb21wdCkuaW5wdXRfaWRzJTBBJTBBJTIzJTIwTm90ZSUyMHRoYXQlMjB3ZSUyMGNhbm5vdCUyMGFkZCUyMCUyMiU3QmV4dHJhX2lkXy4uLiU3RCUyMiUyMHRvJTIwdGhlJTIwc3RyaW5nJTIwZGlyZWN0bHklMEElMjMlMjBhcyUyMHRoZSUyMEJ5dGUlMjB0b2tlbml6ZXIlMjB3b3VsZCUyMGluY29ycmVjdGx5JTIwbWVyZ2UlMjB0aGUlMjB0b2tlbnMlMEElMjMlMjBGb3IlMjBCeVQ1JTJDJTIwd2UlMjBuZWVkJTIwdG8lMjB3b3JrJTIwZGlyZWN0bHklMjBvbiUyMHRoZSUyMGNoYXJhY3RlciUyMGxldmVsJTBBJTIzJTIwQ29udHJhcnklMjB0byUyMFQ1JTJDJTIwQnlUNSUyMGRvZXMlMjBub3QlMjB1c2UlMjBzZW50aW5lbCUyMHRva2VucyUyMGZvciUyMG1hc2tpbmclMkMlMjBidXQlMjBpbnN0ZWFkJTBBJTIzJTIwdXNlcyUyMGZpbmFsJTIwdXRmJTIwY2hhcmFjdGVyJTIwaWRzLiUwQSUyMyUyMFVURi04JTIwaXMlMjByZXByZXNlbnRlZCUyMGJ5JTIwOCUyMGJpdHMlMjBhbmQlMjBCeVQ1JTIwaGFzJTIwMyUyMHNwZWNpYWwlMjB0b2tlbnMuJTBBJTIzJTIwJTNEJTNFJTIwVGhlcmUlMjBhcmUlMjAyKio4JTJCMiUyMCUzRCUyMDI1OSUyMGlucHV0JTIwaWRzJTIwYW5kJTIwbWFzayUyMHRva2VucyUyMGNvdW50JTIwZG93biUyMGZyb20lMjBpbmRleCUyMDI1OC4lMEElMjMlMjAlM0QlM0UlMjBtYXNrJTIwdG8lMjAlMjJUaGUlMjBkb2clMjAlNUIyNTglNURhJTIwYmFsbCUyMCU1QjI1NyU1RHBhcmsuJTIyJTBBJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9yY2gudGVuc29yKCU1QmlucHV0X2lkcyU1QiUzQTglNUQlMjAlMkIlMjAlNUIyNTglNUQlMjAlMkIlMjBpbnB1dF9pZHMlNUIxNCUzQTIxJTVEJTIwJTJCJTIwJTVCMjU3JTVEJTIwJTJCJTIwaW5wdXRfaWRzJTVCMjglM0ElNUQlNUQpJTBBaW5wdXRfaWRzJTBBJTBBJTIzJTIwQnlUNSUyMHByb2R1Y2VzJTIwb25seSUyMG9uZSUyMGNoYXIlMjBhdCUyMGElMjB0aW1lJTIwc28lMjB3ZSUyMG5lZWQlMjB0byUyMHByb2R1Y2UlMjBtYW55JTIwbW9yZSUyMG91dHB1dCUyMGNoYXJhY3RlcnMlMjBoZXJlJTIwLSUzRSUyMHNldCUyMCU2MG1heF9sZW5ndGglM0QxMDAlNjAuJTBBb3V0cHV0X2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKGlucHV0X2lkcyUyQyUyMG1heF9sZW5ndGglM0QxMDApJTVCMCU1RC50b2xpc3QoKSUwQW91dHB1dF9pZHMlMEElMEElMjMlMjAlNUUtJTIwTm90ZSUyMGhvdyUyMDI1OCUyMGRlc2NlbmRzJTIwdG8lMjAyNTclMkMlMjAyNTYlMkMlMjAyNTUlMEElMEElMjMlMjBOb3clMjB3ZSUyMG5lZWQlMjB0byUyMHNwbGl0JTIwb24lMjB0aGUlMjBzZW50aW5lbCUyMHRva2VucyUyQyUyMGxldCdzJTIwd3JpdGUlMjBhJTIwc2hvcnQlMjBsb29wJTIwZm9yJTIwdGhpcyUwQW91dHB1dF9pZHNfbGlzdCUyMCUzRCUyMCU1QiU1RCUwQXN0YXJ0X3Rva2VuJTIwJTNEJTIwMCUwQXNlbnRpbmVsX3Rva2VuJTIwJTNEJTIwMjU4JTBBd2hpbGUlMjBzZW50aW5lbF90b2tlbiUyMGluJTIwb3V0cHV0X2lkcyUzQSUwQSUyMCUyMCUyMCUyMHNwbGl0X2lkeCUyMCUzRCUyMG91dHB1dF9pZHMuaW5kZXgoc2VudGluZWxfdG9rZW4pJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2lkc19saXN0LmFwcGVuZChvdXRwdXRfaWRzJTVCc3RhcnRfdG9rZW4lM0FzcGxpdF9pZHglNUQpJTBBJTIwJTIwJTIwJTIwc3RhcnRfdG9rZW4lMjAlM0QlMjBzcGxpdF9pZHglMEElMjAlMjAlMjAlMjBzZW50aW5lbF90b2tlbiUyMC0lM0QlMjAxJTBBJTBBb3V0cHV0X2lkc19saXN0LmFwcGVuZChvdXRwdXRfaWRzJTVCc3RhcnRfdG9rZW4lM0ElNUQpJTBBb3V0cHV0X3N0cmluZyUyMCUzRCUyMHRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0cHV0X2lkc19saXN0KSUwQW91dHB1dF9zdHJpbmc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSeq2SeqLM
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google/byt5-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;google/byt5-base&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids_prompt = <span class="hljs-string">&quot;The dog chases a ball in the park.&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(input_ids_prompt).input_ids

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Note that we cannot add &quot;{extra_id_...}&quot; to the string directly</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># as the Byte tokenizer would incorrectly merge the tokens</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># For ByT5, we need to work directly on the character level</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Contrary to T5, ByT5 does not use sentinel tokens for masking, but instead</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># uses final utf character ids.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># UTF-8 is represented by 8 bits and ByT5 has 3 special tokens.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># =&gt; There are 2**8+2 = 259 input ids and mask tokens count down from index 258.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># =&gt; mask to &quot;The dog [258]a ball [257]park.&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = torch.tensor([input_ids[:<span class="hljs-number">8</span>] + [<span class="hljs-number">258</span>] + input_ids[<span class="hljs-number">14</span>:<span class="hljs-number">21</span>] + [<span class="hljs-number">257</span>] + input_ids[<span class="hljs-number">28</span>:]])
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids
tensor([[ <span class="hljs-number">87</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">103</span>, <span class="hljs-number">114</span>, <span class="hljs-number">106</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">258</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">100</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">101</span>, <span class="hljs-number">100</span>, <span class="hljs-number">111</span>, <span class="hljs-number">111</span>, <span class="hljs-number">257</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">115</span>, <span class="hljs-number">100</span>, <span class="hljs-number">117</span>, <span class="hljs-number">110</span>,  <span class="hljs-number">49</span>,   <span class="hljs-number">1</span>]])

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># ByT5 produces only one char at a time so we need to produce many more output characters here -&gt; set \`max_length=100\`.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>output_ids = model.generate(input_ids, max_length=<span class="hljs-number">100</span>)[<span class="hljs-number">0</span>].tolist()
<span class="hljs-meta">&gt;&gt;&gt; </span>output_ids
[<span class="hljs-number">0</span>, <span class="hljs-number">258</span>, <span class="hljs-number">108</span>, <span class="hljs-number">118</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">119</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">114</span>, <span class="hljs-number">113</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">122</span>, <span class="hljs-number">107</span>, <span class="hljs-number">114</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">103</span>, <span class="hljs-number">114</span>, <span class="hljs-number">104</span>, <span class="hljs-number">118</span>, <span class="hljs-number">257</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">108</span>, <span class="hljs-number">113</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">119</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">103</span>, <span class="hljs-number">108</span>, <span class="hljs-number">118</span>, <span class="hljs-number">102</span>, <span class="hljs-number">114</span>, <span class="hljs-number">256</span>, <span class="hljs-number">108</span>, <span class="hljs-number">113</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">119</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>, <span class="hljs-number">35</span>, <span class="hljs-number">115</span>, <span class="hljs-number">100</span>, <span class="hljs-number">117</span>, <span class="hljs-number">110</span>,  <span class="hljs-number">49</span>,  <span class="hljs-number">35</span>,  <span class="hljs-number">87</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">103</span>, <span class="hljs-number">114</span>, <span class="hljs-number">106</span>, <span class="hljs-number">35</span>, <span class="hljs-number">108</span>, <span class="hljs-number">118</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">119</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">114</span>, <span class="hljs-number">113</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">122</span>, <span class="hljs-number">107</span>, <span class="hljs-number">114</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">103</span>, <span class="hljs-number">114</span>, <span class="hljs-number">104</span>, <span class="hljs-number">118</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">100</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">101</span>, <span class="hljs-number">100</span>, <span class="hljs-number">111</span>, <span class="hljs-number">111</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">108</span>, <span class="hljs-number">113</span>, <span class="hljs-number">255</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">108</span>, <span class="hljs-number">113</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">119</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">115</span>, <span class="hljs-number">100</span>, <span class="hljs-number">117</span>, <span class="hljs-number">110</span>,  <span class="hljs-number">49</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># ^- Note how 258 descends to 257, 256, 255</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Now we need to split on the sentinel tokens, let&#x27;s write a short loop for this</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>output_ids_list = []
<span class="hljs-meta">&gt;&gt;&gt; </span>start_token = <span class="hljs-number">0</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>sentinel_token = <span class="hljs-number">258</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">while</span> sentinel_token <span class="hljs-keyword">in</span> output_ids:
<span class="hljs-meta">... </span>    split_idx = output_ids.index(sentinel_token)
<span class="hljs-meta">... </span>    output_ids_list.append(output_ids[start_token:split_idx])
<span class="hljs-meta">... </span>    start_token = split_idx
<span class="hljs-meta">... </span>    sentinel_token -= <span class="hljs-number">1</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>output_ids_list.append(output_ids[start_token:])
<span class="hljs-meta">&gt;&gt;&gt; </span>output_string = tokenizer.batch_decode(output_ids_list)
<span class="hljs-meta">&gt;&gt;&gt; </span>output_string
[<span class="hljs-string">&#x27;&lt;pad&gt;&#x27;</span>, <span class="hljs-string">&#x27;is the one who does&#x27;</span>, <span class="hljs-string">&#x27; in the disco&#x27;</span>, <span class="hljs-string">&#x27;in the park. The dog is the one who does a ball in&#x27;</span>, <span class="hljs-string">&#x27; in the park.&#x27;</span>]`}}),L=new he({}),Y=new Es({props:{name:"class transformers.ByT5Tokenizer",anchor:"transformers.ByT5Tokenizer",parameters:[{name:"eos_token",val:" = '</s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"extra_ids",val:" = 125"},{name:"additional_special_tokens",val:" = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.ByT5Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the end of sequence.
The token used is the <code>sep_token</code>.</p>

					</div>`,name:"eos_token"},{anchor:"transformers.ByT5Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.ByT5Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;pad&gt;&quot;</code>) &#x2014;
The token used for padding, for example when batching sequences of different lengths.`,name:"pad_token"},{anchor:"transformers.ByT5Tokenizer.extra_ids",description:`<strong>extra_ids</strong> (<code>int</code>, <em>optional</em>, defaults to 100) &#x2014;
Add a number of extra ids added to the end of the vocabulary for use as sentinels. These tokens are
accessible as &#x201C;<extra<em>id{%d}&gt;&#x201D; where &#x201D;{%d}&#x201D; is a number between 0 and extra_ids-1. Extra tokens are
indexed from the end of the vocabulary up to beginning (&#x201C;<extra_id_0>&#x201D; is the last token in the vocabulary
like in ByT5 preprocessing see
<a href="https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117" rel="nofollow">here</a>).</extra_id_0></extra<em>`,name:"extra_ids"},{anchor:"transformers.ByT5Tokenizer.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
Additional special tokens used by the tokenizer.`,name:"additional_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/byt5/tokenization_byt5.py#L28"}}),P=new Es({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.ByT5Tokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.ByT5Tokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.ByT5Tokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/byt5/tokenization_byt5.py#L179",returnDescription:`
<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),K=new Es({props:{name:"convert_tokens_to_string",anchor:"transformers.ByT5Tokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/byt5/tokenization_byt5.py#L230"}}),ss=new Es({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.ByT5Tokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.ByT5Tokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.ByT5Tokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/byt5/tokenization_byt5.py#L157",returnDescription:`
<p>List of zeros.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),es=new Es({props:{name:"get_special_tokens_mask",anchor:"transformers.ByT5Tokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.ByT5Tokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.ByT5Tokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.ByT5Tokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/byt5/tokenization_byt5.py#L118",returnDescription:`
<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),{c(){_=a("meta"),Ws=i(),U=a("h1"),I=a("a"),fs=a("span"),b(N.$$.fragment),ue=i(),gs=a("span"),be=r("ByT5"),Cs=i(),J=a("h2"),G=a("a"),js=a("span"),b(q.$$.fragment),ye=i(),Ts=a("span"),fe=r("Overview"),Rs=i(),z=a("p"),ge=r("The ByT5 model was presented in "),$=a("a"),je=r("ByT5: Towards a token-free future with pre-trained byte-to-byte models"),Te=r(` by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir
Kale, Adam Roberts, Colin Raffel.`),Vs=i(),as=a("p"),Me=r("The abstract from the paper is the following:"),Ns=i(),ts=a("p"),Ms=a("em"),ke=r(`Most widely-used pre-trained language models operate on sequences of tokens corresponding to word or subword units.
Encoding text as a sequence of tokens requires a tokenizer, which is typically created as an independent artifact from
the model. Token-free models that instead operate directly on raw text (bytes or characters) have many benefits: they
can process text in any language out of the box, they are more robust to noise, and they minimize technical debt by
removing complex and error-prone text preprocessing pipelines. Since byte or character sequences are longer than token
sequences, past work on token-free models has often introduced new model architectures designed to amortize the cost of
operating directly on raw text. In this paper, we show that a standard Transformer architecture can be used with
minimal modifications to process byte sequences. We carefully characterize the trade-offs in terms of parameter count,
training FLOPs, and inference speed, and show that byte-level models are competitive with their token-level
counterparts. We also demonstrate that byte-level models are significantly more robust to noise and perform better on
tasks that are sensitive to spelling and pronunciation. As part of our contribution, we release a new set of
pre-trained byte-level Transformer models based on the T5 architecture, as well as all code and data used in our
experiments.`),qs=i(),M=a("p"),we=r("This model was contributed by "),X=a("a"),_e=r("patrickvonplaten"),Ue=r(`. The original code can be
found `),Q=a("a"),Je=r("here"),ve=r("."),$s=i(),x=a("p"),Be=r("ByT5\u2019s architecture is based on the T5v1.1 model, so one can refer to "),ls=a("a"),Ie=r("T5v1.1\u2019s documentation page"),Ge=r(`. They
only differ in how inputs should be prepared for the model, see the code examples below.`),Xs=i(),rs=a("p"),ze=r(`Since ByT5 was pre-trained unsupervisedly, there\u2019s no real advantage to using a task prefix during single-task
fine-tuning. If you are doing multi-task fine-tuning, you should use a prefix.`),Qs=i(),v=a("h3"),Z=a("a"),ks=a("span"),b(F.$$.fragment),xe=i(),ws=a("span"),Ze=r("Example"),Fs=i(),os=a("p"),Ee=r("ByT5 works on raw UTF-8 bytes, so it can be used without a tokenizer:"),Ss=i(),b(S.$$.fragment),As=i(),ps=a("p"),We=r("For batched inference and training it is however recommended to make use of the tokenizer:"),Hs=i(),b(A.$$.fragment),Ls=i(),k=a("p"),Ce=r("Similar to "),is=a("a"),Re=r("T5"),Ve=r(`, ByT5 was trained on the span-mask denoising task. However,
since the model works directly on characters, the pretraining task is a bit
different. Let\u2019s corrupt some characters of the
input sentence `),_s=a("code"),Ne=r('"The dog chases a ball in the park."'),qe=r(` and ask ByT5 to predict them
for us.`),Ys=i(),b(H.$$.fragment),Ds=i(),B=a("h2"),E=a("a"),Us=a("span"),b(L.$$.fragment),$e=i(),Js=a("span"),Xe=r("ByT5Tokenizer"),Ps=i(),h=a("div"),b(Y.$$.fragment),Qe=i(),vs=a("p"),Fe=r("Construct a ByT5 tokenizer. ByT5 simply uses raw bytes utf-8 encoding."),Se=i(),D=a("p"),Ae=r("This tokenizer inherits from "),cs=a("a"),He=r("PreTrainedTokenizer"),Le=r(` which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`),Ye=i(),w=a("div"),b(P.$$.fragment),De=i(),Bs=a("p"),Pe=r(`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A sequence has the following format:`),Oe=i(),O=a("ul"),ds=a("li"),Ke=r("single sequence: "),Is=a("code"),sn=r("X </s>"),en=i(),ms=a("li"),nn=r("pair of sequences: "),Gs=a("code"),an=r("A </s> B </s>"),tn=i(),W=a("div"),b(K.$$.fragment),ln=i(),zs=a("p"),rn=r("Converts a sequence of tokens (string) in a single string."),on=i(),C=a("div"),b(ss.$$.fragment),pn=i(),xs=a("p"),cn=r(`Create a mask from the two sequences passed to be used in a sequence-pair classification task. ByT5 does not
make use of token type ids, therefore a list of zeros is returned.`),dn=i(),R=a("div"),b(es.$$.fragment),mn=i(),ns=a("p"),hn=r(`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `),Zs=a("code"),un=r("prepare_for_model"),bn=r(" method."),Os=i(),V=a("p"),yn=r("See "),hs=a("a"),fn=r("ByT5Tokenizer"),gn=r(" for all details."),this.h()},l(s){const p=la('[data-svelte="svelte-1phssyn"]',document.head);_=t(p,"META",{name:!0,content:!0}),p.forEach(n),Ws=c(s),U=t(s,"H1",{class:!0});var se=l(U);I=t(se,"A",{id:!0,class:!0,href:!0});var wn=l(I);fs=t(wn,"SPAN",{});var _n=l(fs);y(N.$$.fragment,_n),_n.forEach(n),wn.forEach(n),ue=c(se),gs=t(se,"SPAN",{});var Un=l(gs);be=o(Un,"ByT5"),Un.forEach(n),se.forEach(n),Cs=c(s),J=t(s,"H2",{class:!0});var ee=l(J);G=t(ee,"A",{id:!0,class:!0,href:!0});var Jn=l(G);js=t(Jn,"SPAN",{});var vn=l(js);y(q.$$.fragment,vn),vn.forEach(n),Jn.forEach(n),ye=c(ee),Ts=t(ee,"SPAN",{});var Bn=l(Ts);fe=o(Bn,"Overview"),Bn.forEach(n),ee.forEach(n),Rs=c(s),z=t(s,"P",{});var ne=l(z);ge=o(ne,"The ByT5 model was presented in "),$=t(ne,"A",{href:!0,rel:!0});var In=l($);je=o(In,"ByT5: Towards a token-free future with pre-trained byte-to-byte models"),In.forEach(n),Te=o(ne,` by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir
Kale, Adam Roberts, Colin Raffel.`),ne.forEach(n),Vs=c(s),as=t(s,"P",{});var Gn=l(as);Me=o(Gn,"The abstract from the paper is the following:"),Gn.forEach(n),Ns=c(s),ts=t(s,"P",{});var zn=l(ts);Ms=t(zn,"EM",{});var xn=l(Ms);ke=o(xn,`Most widely-used pre-trained language models operate on sequences of tokens corresponding to word or subword units.
Encoding text as a sequence of tokens requires a tokenizer, which is typically created as an independent artifact from
the model. Token-free models that instead operate directly on raw text (bytes or characters) have many benefits: they
can process text in any language out of the box, they are more robust to noise, and they minimize technical debt by
removing complex and error-prone text preprocessing pipelines. Since byte or character sequences are longer than token
sequences, past work on token-free models has often introduced new model architectures designed to amortize the cost of
operating directly on raw text. In this paper, we show that a standard Transformer architecture can be used with
minimal modifications to process byte sequences. We carefully characterize the trade-offs in terms of parameter count,
training FLOPs, and inference speed, and show that byte-level models are competitive with their token-level
counterparts. We also demonstrate that byte-level models are significantly more robust to noise and perform better on
tasks that are sensitive to spelling and pronunciation. As part of our contribution, we release a new set of
pre-trained byte-level Transformer models based on the T5 architecture, as well as all code and data used in our
experiments.`),xn.forEach(n),zn.forEach(n),qs=c(s),M=t(s,"P",{});var us=l(M);we=o(us,"This model was contributed by "),X=t(us,"A",{href:!0,rel:!0});var Zn=l(X);_e=o(Zn,"patrickvonplaten"),Zn.forEach(n),Ue=o(us,`. The original code can be
found `),Q=t(us,"A",{href:!0,rel:!0});var En=l(Q);Je=o(En,"here"),En.forEach(n),ve=o(us,"."),us.forEach(n),$s=c(s),x=t(s,"P",{});var ae=l(x);Be=o(ae,"ByT5\u2019s architecture is based on the T5v1.1 model, so one can refer to "),ls=t(ae,"A",{href:!0});var Wn=l(ls);Ie=o(Wn,"T5v1.1\u2019s documentation page"),Wn.forEach(n),Ge=o(ae,`. They
only differ in how inputs should be prepared for the model, see the code examples below.`),ae.forEach(n),Xs=c(s),rs=t(s,"P",{});var Cn=l(rs);ze=o(Cn,`Since ByT5 was pre-trained unsupervisedly, there\u2019s no real advantage to using a task prefix during single-task
fine-tuning. If you are doing multi-task fine-tuning, you should use a prefix.`),Cn.forEach(n),Qs=c(s),v=t(s,"H3",{class:!0});var te=l(v);Z=t(te,"A",{id:!0,class:!0,href:!0});var Rn=l(Z);ks=t(Rn,"SPAN",{});var Vn=l(ks);y(F.$$.fragment,Vn),Vn.forEach(n),Rn.forEach(n),xe=c(te),ws=t(te,"SPAN",{});var Nn=l(ws);Ze=o(Nn,"Example"),Nn.forEach(n),te.forEach(n),Fs=c(s),os=t(s,"P",{});var qn=l(os);Ee=o(qn,"ByT5 works on raw UTF-8 bytes, so it can be used without a tokenizer:"),qn.forEach(n),Ss=c(s),y(S.$$.fragment,s),As=c(s),ps=t(s,"P",{});var $n=l(ps);We=o($n,"For batched inference and training it is however recommended to make use of the tokenizer:"),$n.forEach(n),Hs=c(s),y(A.$$.fragment,s),Ls=c(s),k=t(s,"P",{});var bs=l(k);Ce=o(bs,"Similar to "),is=t(bs,"A",{href:!0});var Xn=l(is);Re=o(Xn,"T5"),Xn.forEach(n),Ve=o(bs,`, ByT5 was trained on the span-mask denoising task. However,
since the model works directly on characters, the pretraining task is a bit
different. Let\u2019s corrupt some characters of the
input sentence `),_s=t(bs,"CODE",{});var Qn=l(_s);Ne=o(Qn,'"The dog chases a ball in the park."'),Qn.forEach(n),qe=o(bs,` and ask ByT5 to predict them
for us.`),bs.forEach(n),Ys=c(s),y(H.$$.fragment,s),Ds=c(s),B=t(s,"H2",{class:!0});var le=l(B);E=t(le,"A",{id:!0,class:!0,href:!0});var Fn=l(E);Us=t(Fn,"SPAN",{});var Sn=l(Us);y(L.$$.fragment,Sn),Sn.forEach(n),Fn.forEach(n),$e=c(le),Js=t(le,"SPAN",{});var An=l(Js);Xe=o(An,"ByT5Tokenizer"),An.forEach(n),le.forEach(n),Ps=c(s),h=t(s,"DIV",{class:!0});var u=l(h);y(Y.$$.fragment,u),Qe=c(u),vs=t(u,"P",{});var Hn=l(vs);Fe=o(Hn,"Construct a ByT5 tokenizer. ByT5 simply uses raw bytes utf-8 encoding."),Hn.forEach(n),Se=c(u),D=t(u,"P",{});var re=l(D);Ae=o(re,"This tokenizer inherits from "),cs=t(re,"A",{href:!0});var Ln=l(cs);He=o(Ln,"PreTrainedTokenizer"),Ln.forEach(n),Le=o(re,` which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`),re.forEach(n),Ye=c(u),w=t(u,"DIV",{class:!0});var ys=l(w);y(P.$$.fragment,ys),De=c(ys),Bs=t(ys,"P",{});var Yn=l(Bs);Pe=o(Yn,`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A sequence has the following format:`),Yn.forEach(n),Oe=c(ys),O=t(ys,"UL",{});var oe=l(O);ds=t(oe,"LI",{});var jn=l(ds);Ke=o(jn,"single sequence: "),Is=t(jn,"CODE",{});var Dn=l(Is);sn=o(Dn,"X </s>"),Dn.forEach(n),jn.forEach(n),en=c(oe),ms=t(oe,"LI",{});var Tn=l(ms);nn=o(Tn,"pair of sequences: "),Gs=t(Tn,"CODE",{});var Pn=l(Gs);an=o(Pn,"A </s> B </s>"),Pn.forEach(n),Tn.forEach(n),oe.forEach(n),ys.forEach(n),tn=c(u),W=t(u,"DIV",{class:!0});var pe=l(W);y(K.$$.fragment,pe),ln=c(pe),zs=t(pe,"P",{});var On=l(zs);rn=o(On,"Converts a sequence of tokens (string) in a single string."),On.forEach(n),pe.forEach(n),on=c(u),C=t(u,"DIV",{class:!0});var ie=l(C);y(ss.$$.fragment,ie),pn=c(ie),xs=t(ie,"P",{});var Kn=l(xs);cn=o(Kn,`Create a mask from the two sequences passed to be used in a sequence-pair classification task. ByT5 does not
make use of token type ids, therefore a list of zeros is returned.`),Kn.forEach(n),ie.forEach(n),dn=c(u),R=t(u,"DIV",{class:!0});var ce=l(R);y(es.$$.fragment,ce),mn=c(ce),ns=t(ce,"P",{});var de=l(ns);hn=o(de,`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `),Zs=t(de,"CODE",{});var sa=l(Zs);un=o(sa,"prepare_for_model"),sa.forEach(n),bn=o(de," method."),de.forEach(n),ce.forEach(n),u.forEach(n),Os=c(s),V=t(s,"P",{});var me=l(V);yn=o(me,"See "),hs=t(me,"A",{href:!0});var ea=l(hs);fn=o(ea,"ByT5Tokenizer"),ea.forEach(n),gn=o(me," for all details."),me.forEach(n),this.h()},h(){d(_,"name","hf:doc:metadata"),d(_,"content",JSON.stringify(ia)),d(I,"id","byt5"),d(I,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(I,"href","#byt5"),d(U,"class","relative group"),d(G,"id","overview"),d(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(G,"href","#overview"),d(J,"class","relative group"),d($,"href","https://arxiv.org/abs/2105.13626"),d($,"rel","nofollow"),d(X,"href","https://huggingface.co/patrickvonplaten"),d(X,"rel","nofollow"),d(Q,"href","https://github.com/google-research/byt5"),d(Q,"rel","nofollow"),d(ls,"href","t5v1.1"),d(Z,"id","example"),d(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Z,"href","#example"),d(v,"class","relative group"),d(is,"href","t5"),d(E,"id","transformers.ByT5Tokenizer"),d(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(E,"href","#transformers.ByT5Tokenizer"),d(B,"class","relative group"),d(cs,"href","/docs/transformers/v4.33.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizer"),d(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(h,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(hs,"href","/docs/transformers/v4.33.0/en/model_doc/byt5#transformers.ByT5Tokenizer")},m(s,p){e(document.head,_),m(s,Ws,p),m(s,U,p),e(U,I),e(I,fs),f(N,fs,null),e(U,ue),e(U,gs),e(gs,be),m(s,Cs,p),m(s,J,p),e(J,G),e(G,js),f(q,js,null),e(J,ye),e(J,Ts),e(Ts,fe),m(s,Rs,p),m(s,z,p),e(z,ge),e(z,$),e($,je),e(z,Te),m(s,Vs,p),m(s,as,p),e(as,Me),m(s,Ns,p),m(s,ts,p),e(ts,Ms),e(Ms,ke),m(s,qs,p),m(s,M,p),e(M,we),e(M,X),e(X,_e),e(M,Ue),e(M,Q),e(Q,Je),e(M,ve),m(s,$s,p),m(s,x,p),e(x,Be),e(x,ls),e(ls,Ie),e(x,Ge),m(s,Xs,p),m(s,rs,p),e(rs,ze),m(s,Qs,p),m(s,v,p),e(v,Z),e(Z,ks),f(F,ks,null),e(v,xe),e(v,ws),e(ws,Ze),m(s,Fs,p),m(s,os,p),e(os,Ee),m(s,Ss,p),f(S,s,p),m(s,As,p),m(s,ps,p),e(ps,We),m(s,Hs,p),f(A,s,p),m(s,Ls,p),m(s,k,p),e(k,Ce),e(k,is),e(is,Re),e(k,Ve),e(k,_s),e(_s,Ne),e(k,qe),m(s,Ys,p),f(H,s,p),m(s,Ds,p),m(s,B,p),e(B,E),e(E,Us),f(L,Us,null),e(B,$e),e(B,Js),e(Js,Xe),m(s,Ps,p),m(s,h,p),f(Y,h,null),e(h,Qe),e(h,vs),e(vs,Fe),e(h,Se),e(h,D),e(D,Ae),e(D,cs),e(cs,He),e(D,Le),e(h,Ye),e(h,w),f(P,w,null),e(w,De),e(w,Bs),e(Bs,Pe),e(w,Oe),e(w,O),e(O,ds),e(ds,Ke),e(ds,Is),e(Is,sn),e(O,en),e(O,ms),e(ms,nn),e(ms,Gs),e(Gs,an),e(h,tn),e(h,W),f(K,W,null),e(W,ln),e(W,zs),e(zs,rn),e(h,on),e(h,C),f(ss,C,null),e(C,pn),e(C,xs),e(xs,cn),e(h,dn),e(h,R),f(es,R,null),e(R,mn),e(R,ns),e(ns,hn),e(ns,Zs),e(Zs,un),e(ns,bn),m(s,Os,p),m(s,V,p),e(V,yn),e(V,hs),e(hs,fn),e(V,gn),Ks=!0},p:ra,i(s){Ks||(g(N.$$.fragment,s),g(q.$$.fragment,s),g(F.$$.fragment,s),g(S.$$.fragment,s),g(A.$$.fragment,s),g(H.$$.fragment,s),g(L.$$.fragment,s),g(Y.$$.fragment,s),g(P.$$.fragment,s),g(K.$$.fragment,s),g(ss.$$.fragment,s),g(es.$$.fragment,s),Ks=!0)},o(s){j(N.$$.fragment,s),j(q.$$.fragment,s),j(F.$$.fragment,s),j(S.$$.fragment,s),j(A.$$.fragment,s),j(H.$$.fragment,s),j(L.$$.fragment,s),j(Y.$$.fragment,s),j(P.$$.fragment,s),j(K.$$.fragment,s),j(ss.$$.fragment,s),j(es.$$.fragment,s),Ks=!1},d(s){n(_),s&&n(Ws),s&&n(U),T(N),s&&n(Cs),s&&n(J),T(q),s&&n(Rs),s&&n(z),s&&n(Vs),s&&n(as),s&&n(Ns),s&&n(ts),s&&n(qs),s&&n(M),s&&n($s),s&&n(x),s&&n(Xs),s&&n(rs),s&&n(Qs),s&&n(v),T(F),s&&n(Fs),s&&n(os),s&&n(Ss),T(S,s),s&&n(As),s&&n(ps),s&&n(Hs),T(A,s),s&&n(Ls),s&&n(k),s&&n(Ys),T(H,s),s&&n(Ds),s&&n(B),T(L),s&&n(Ps),s&&n(h),T(Y),T(P),T(K),T(ss),T(es),s&&n(Os),s&&n(V)}}}const ia={local:"byt5",sections:[{local:"overview",sections:[{local:"example",title:"Example"}],title:"Overview"},{local:"transformers.ByT5Tokenizer",title:"ByT5Tokenizer"}],title:"ByT5"};function ca(kn){return oa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ba extends na{constructor(_){super();aa(this,_,ca,pa,ta,{})}}export{ba as default,ia as metadata};
