import{S as Zl,i as Bl,s as Fl,e as s,k as c,w as g,t as r,M as Rl,c as n,d as a,m as d,a as o,x as b,h as l,b as h,G as t,g as p,y as k,q as _,o as y,B as v,v as Sl,L as Ll}from"../../chunks/vendor-hf-doc-builder.js";import{D as Qe}from"../../chunks/Docstring-hf-doc-builder.js";import{C as et}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Ke}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as Il}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function Wl(tt){let m,z,w,u,T;return u=new et({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyME5sbGJUb2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBObGxiVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJmYWNlYm9vayUyRm5sbGItMjAwLWRpc3RpbGxlZC02MDBNJTIyJTJDJTIwc3JjX2xhbmclM0QlMjJlbmdfTGF0biUyMiUyQyUyMHRndF9sYW5nJTNEJTIyZnJhX0xhdG4lMjIlMEEpJTBBZXhhbXBsZV9lbmdsaXNoX3BocmFzZSUyMCUzRCUyMCUyMiUyMFVOJTIwQ2hpZWYlMjBTYXlzJTIwVGhlcmUlMjBJcyUyME5vJTIwTWlsaXRhcnklMjBTb2x1dGlvbiUyMGluJTIwU3lyaWElMjIlMEFleHBlY3RlZF90cmFuc2xhdGlvbl9mcmVuY2glMjAlM0QlMjAlMjJMZSUyMGNoZWYlMjBkZSUyMGwnT05VJTIwYWZmaXJtZSUyMHF1J2lsJTIwbid5JTIwYSUyMHBhcyUyMGRlJTIwc29sdXRpb24lMjBtaWxpdGFpcmUlMjBlbiUyMFN5cmllLiUyMiUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihleGFtcGxlX2VuZ2xpc2hfcGhyYXNlJTJDJTIwdGV4dF90YXJnZXQlM0RleHBlY3RlZF90cmFuc2xhdGlvbl9mcmVuY2glMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> NllbTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = NllbTokenizer.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;facebook/nllb-200-distilled-600M&quot;</span>, src_lang=<span class="hljs-string">&quot;eng_Latn&quot;</span>, tgt_lang=<span class="hljs-string">&quot;fra_Latn&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>example_english_phrase = <span class="hljs-string">&quot; UN Chief Says There Is No Military Solution in Syria&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>expected_translation_french = <span class="hljs-string">&quot;Le chef de l&#x27;ONU affirme qu&#x27;il n&#x27;y a pas de solution militaire en Syrie.&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(example_english_phrase, text_target=expected_translation_french, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)`}}),{c(){m=s("p"),z=r("Examples:"),w=c(),g(u.$$.fragment)},l(f){m=n(f,"P",{});var M=o(m);z=l(M,"Examples:"),M.forEach(a),w=d(f),b(u.$$.fragment,f)},m(f,M){p(f,m,M),t(m,z),p(f,w,M),k(u,f,M),T=!0},p:Ll,i(f){T||(_(u.$$.fragment,f),T=!0)},o(f){y(u.$$.fragment,f),T=!1},d(f){f&&a(m),f&&a(w),v(u,f)}}}function Cl(tt){let m,z,w,u,T;return u=new et({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyME5sbGJUb2tlbml6ZXJGYXN0JTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwTmxsYlRva2VuaXplckZhc3QuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmZhY2Vib29rJTJGbmxsYi0yMDAtZGlzdGlsbGVkLTYwME0lMjIlMkMlMjBzcmNfbGFuZyUzRCUyMmVuZ19MYXRuJTIyJTJDJTIwdGd0X2xhbmclM0QlMjJmcmFfTGF0biUyMiUwQSklMEFleGFtcGxlX2VuZ2xpc2hfcGhyYXNlJTIwJTNEJTIwJTIyJTIwVU4lMjBDaGllZiUyMFNheXMlMjBUaGVyZSUyMElzJTIwTm8lMjBNaWxpdGFyeSUyMFNvbHV0aW9uJTIwaW4lMjBTeXJpYSUyMiUwQWV4cGVjdGVkX3RyYW5zbGF0aW9uX2ZyZW5jaCUyMCUzRCUyMCUyMkxlJTIwY2hlZiUyMGRlJTIwbCdPTlUlMjBhZmZpcm1lJTIwcXUnaWwlMjBuJ3klMjBhJTIwcGFzJTIwZGUlMjBzb2x1dGlvbiUyMG1pbGl0YWlyZSUyMGVuJTIwU3lyaWUuJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGVfZW5nbGlzaF9waHJhc2UlMkMlMjB0ZXh0X3RhcmdldCUzRGV4cGVjdGVkX3RyYW5zbGF0aW9uX2ZyZW5jaCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> NllbTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = NllbTokenizerFast.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;facebook/nllb-200-distilled-600M&quot;</span>, src_lang=<span class="hljs-string">&quot;eng_Latn&quot;</span>, tgt_lang=<span class="hljs-string">&quot;fra_Latn&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>example_english_phrase = <span class="hljs-string">&quot; UN Chief Says There Is No Military Solution in Syria&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>expected_translation_french = <span class="hljs-string">&quot;Le chef de l&#x27;ONU affirme qu&#x27;il n&#x27;y a pas de solution militaire en Syrie.&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(example_english_phrase, text_target=expected_translation_french, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)`}}),{c(){m=s("p"),z=r("Examples:"),w=c(),g(u.$$.fragment)},l(f){m=n(f,"P",{});var M=o(m);z=l(M,"Examples:"),M.forEach(a),w=d(f),b(u.$$.fragment,f)},m(f,M){p(f,m,M),t(m,z),p(f,w,M),k(u,f,M),T=!0},p:Ll,i(f){T||(_(u.$$.fragment,f),T=!0)},o(f){y(u.$$.fragment,f),T=!1},d(f){f&&a(m),f&&a(w),v(u,f)}}}function Xl(tt){let m,z,w,u,T,f,M,Jt,js,Na,fe,jt,Ns,zs,za,H,$s,Nt,Us,xs,$a,at,zt,qs,Ua,st,Gs,xa,ue,qa,nt,Is,Ga,me,Ia,ot,Ls,La,ge,Za,q,Zs,be,Bs,Fs,ke,Rs,Ss,Ba,C,O,$t,_e,Ws,Ut,Cs,Fa,Q,Xs,ye,Vs,As,Ra,rt,Ps,Sa,lt,xt,Ds,Wa,it,Ys,Ca,ct,dt,Hs,pt,Os,Xa,G,Qs,ve,Ks,en,we,tn,an,Va,X,K,qt,Me,sn,Gt,nn,Aa,I,on,It,rn,ln,Lt,cn,dn,Pa,L,pn,Zt,hn,fn,Te,un,mn,Da,Ee,Ya,V,ee,Bt,Je,gn,Ft,bn,Ha,Z,kn,Rt,_n,yn,St,vn,wn,Oa,ht,Mn,Qa,je,Ka,A,te,Wt,Ne,Tn,Ct,En,es,ae,Xt,ft,Jn,jn,Vt,ut,Nn,ts,P,se,At,ze,zn,Pt,$n,as,E,$e,Un,Dt,xn,qn,U,Gn,mt,In,Ln,gt,Zn,Bn,Ue,Fn,Rn,Sn,ne,Wn,Yt,Cn,Xn,Ht,Vn,Ot,oe,An,re,Pn,$,xe,Dn,qe,Yn,Qt,Hn,On,Qn,Ge,Ie,Kt,Kn,eo,ea,to,ao,Le,ta,so,no,aa,oo,ro,sa,lo,ss,D,le,na,Ze,io,oa,co,ns,J,Be,po,Y,ho,ra,fo,uo,Fe,mo,go,bo,Re,ko,bt,_o,yo,vo,ie,wo,la,Mo,To,ia,Eo,ca,j,Jo,ce,jo,N,Se,No,da,zo,$o,We,Uo,pa,xo,qo,Go,Ce,Xe,ha,Io,Lo,fa,Zo,Bo,Ve,ua,Fo,Ro,ma,So,Wo,ga,Co,Xo,de,Ae,Vo,ba,Ao,Po,B,Pe,Do,ka,Yo,Ho,De,_a,Oo,Qo,ya,Ko,er,F,Ye,tr,va,ar,sr,He,wa,nr,or,Ma,rr,os;return f=new Ke({}),ue=new et({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyME5sbGJUb2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBObGxiVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm5sbGItMjAwLWRpc3RpbGxlZC02MDBNJTIyKSUwQXRva2VuaXplciglMjJIb3clMjB3YXMlMjB5b3VyJTIwZGF5JTNGJTIyKS5pbnB1dF9pZHMlMEElMEElMjMlMjAyJTNBJTIwJyUzQyUyRnMlM0UnJTBBJTIzJTIwMjU2MDQ3JTIwJTNBJTIwJ2VuZ19MYXRuJw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> NllbTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = NllbTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/nllb-200-distilled-600M&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;How was your day?&quot;</span>).input_ids
[<span class="hljs-number">13374</span>, <span class="hljs-number">1398</span>, <span class="hljs-number">4260</span>, <span class="hljs-number">4039</span>, <span class="hljs-number">248130</span>, <span class="hljs-number">2</span>, <span class="hljs-number">256047</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># 2: &#x27;&lt;/s&gt;&#x27;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># 256047 : &#x27;eng_Latn&#x27;</span>`}}),me=new et({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyME5sbGJUb2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBObGxiVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm5sbGItMjAwLWRpc3RpbGxlZC02MDBNJTIyKSUwQXRva2VuaXplciglMjJIb3clMjB3YXMlMjB5b3VyJTIwZGF5JTNGJTIyKS5pbnB1dF9pZHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> NllbTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = NllbTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/nllb-200-distilled-600M&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;How was your day?&quot;</span>).input_ids
[<span class="hljs-number">256047</span>, <span class="hljs-number">13374</span>, <span class="hljs-number">1398</span>, <span class="hljs-number">4260</span>, <span class="hljs-number">4039</span>, <span class="hljs-number">248130</span>, <span class="hljs-number">2</span>]`}}),ge=new et({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyME5sbGJUb2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBObGxiVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm5sbGItMjAwLWRpc3RpbGxlZC02MDBNJTIyJTJDJTIwbGVnYWN5X2JlaGF2aW91ciUzRFRydWUp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> NllbTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = NllbTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/nllb-200-distilled-600M&quot;</span>, legacy_behaviour=<span class="hljs-literal">True</span>)`}}),_e=new Ke({}),Me=new Ke({}),Ee=new et({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm5sbGItMjAwLWRpc3RpbGxlZC02MDBNJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxMlNlcUxNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm5sbGItMjAwLWRpc3RpbGxlZC02MDBNJTIyKSUwQSUwQWFydGljbGUlMjAlM0QlMjAlMjJVTiUyMENoaWVmJTIwc2F5cyUyMHRoZXJlJTIwaXMlMjBubyUyMG1pbGl0YXJ5JTIwc29sdXRpb24lMjBpbiUyMFN5cmlhJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKGFydGljbGUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQXRyYW5zbGF0ZWRfdG9rZW5zJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoJTBBJTIwJTIwJTIwJTIwKippbnB1dHMlMkMlMjBmb3JjZWRfYm9zX3Rva2VuX2lkJTNEdG9rZW5pemVyLmxhbmdfY29kZV90b19pZCU1QiUyMmZyYV9MYXRuJTIyJTVEJTJDJTIwbWF4X2xlbmd0aCUzRDMwJTBBKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUodHJhbnNsYXRlZF90b2tlbnMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklNUIwJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/nllb-200-distilled-600M&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;facebook/nllb-200-distilled-600M&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>article = <span class="hljs-string">&quot;UN Chief says there is no military solution in Syria&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(article, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>translated_tokens = model.generate(
<span class="hljs-meta">... </span>    **inputs, forced_bos_token_id=tokenizer.lang_code_to_id[<span class="hljs-string">&quot;fra_Latn&quot;</span>], max_length=<span class="hljs-number">30</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(translated_tokens, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
Le chef de l<span class="hljs-string">&#x27;ONU dit qu&#x27;</span>il n<span class="hljs-string">&#x27;y a pas de solution militaire en Syrie</span>`}}),Je=new Ke({}),je=new et({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJmYWNlYm9vayUyRm5sbGItMjAwLWRpc3RpbGxlZC02MDBNJTIyJTJDJTIwdXNlX2F1dGhfdG9rZW4lM0RUcnVlJTJDJTIwc3JjX2xhbmclM0QlMjJyb25fTGF0biUyMiUwQSklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZubGxiLTIwMC1kaXN0aWxsZWQtNjAwTSUyMiUyQyUyMHVzZV9hdXRoX3Rva2VuJTNEVHJ1ZSklMEElMEFhcnRpY2xlJTIwJTNEJTIwJTIyJUM1JTlFZWZ1bCUyME9OVSUyMHNwdW5lJTIwYyVDNCU4MyUyMG51JTIwZXhpc3QlQzQlODMlMjBvJTIwc29sdSVDNSVBM2llJTIwbWlsaXRhciVDNCU4MyUyMCVDMyVBRW4lMjBTaXJpYSUyMiUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihhcnRpY2xlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEF0cmFuc2xhdGVkX3Rva2VucyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCUwQSUyMCUyMCUyMCUyMCoqaW5wdXRzJTJDJTIwZm9yY2VkX2Jvc190b2tlbl9pZCUzRHRva2VuaXplci5sYW5nX2NvZGVfdG9faWQlNUIlMjJkZXVfTGF0biUyMiU1RCUyQyUyMG1heF9sZW5ndGglM0QzMCUwQSklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKHRyYW5zbGF0ZWRfdG9rZW5zJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;facebook/nllb-200-distilled-600M&quot;</span>, use_auth_token=<span class="hljs-literal">True</span>, src_lang=<span class="hljs-string">&quot;ron_Latn&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;facebook/nllb-200-distilled-600M&quot;</span>, use_auth_token=<span class="hljs-literal">True</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>article = <span class="hljs-string">&quot;\u015Eeful ONU spune c\u0103 nu exist\u0103 o solu\u0163ie militar\u0103 \xEEn Siria&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(article, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>translated_tokens = model.generate(
<span class="hljs-meta">... </span>    **inputs, forced_bos_token_id=tokenizer.lang_code_to_id[<span class="hljs-string">&quot;deu_Latn&quot;</span>], max_length=<span class="hljs-number">30</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(translated_tokens, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
UN-Chef sagt, es gibt keine milit\xE4rische L\xF6sung <span class="hljs-keyword">in</span> Syrien`}}),Ne=new Ke({}),ze=new Ke({}),$e=new Qe({props:{name:"class transformers.NllbTokenizer",anchor:"transformers.NllbTokenizer",parameters:[{name:"vocab_file",val:""},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"sep_token",val:" = '</s>'"},{name:"cls_token",val:" = '<s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"mask_token",val:" = '<mask>'"},{name:"tokenizer_file",val:" = None"},{name:"src_lang",val:" = None"},{name:"tgt_lang",val:" = None"},{name:"sp_model_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"additional_special_tokens",val:" = None"},{name:"legacy_behaviour",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.NllbTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to the vocabulary file.`,name:"vocab_file"},{anchor:"transformers.NllbTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the beginning of
sequence. The token used is the <code>cls_token</code>.</p>

					</div>`,name:"bos_token"},{anchor:"transformers.NllbTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the end of sequence.
The token used is the <code>sep_token</code>.</p>

					</div>`,name:"eos_token"},{anchor:"transformers.NllbTokenizer.sep_token",description:`<strong>sep_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
sequence classification or for a text and a question for question answering. It is also used as the last
token of a sequence built with special tokens.`,name:"sep_token"},{anchor:"transformers.NllbTokenizer.cls_token",description:`<strong>cls_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The classifier token which is used when doing sequence classification (classification of the whole sequence
instead of per-token classification). It is the first token of the sequence when built with special tokens.`,name:"cls_token"},{anchor:"transformers.NllbTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.NllbTokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;pad&gt;&quot;</code>) &#x2014;
The token used for padding, for example when batching sequences of different lengths.`,name:"pad_token"},{anchor:"transformers.NllbTokenizer.mask_token",description:`<strong>mask_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;mask&gt;&quot;</code>) &#x2014;
The token used for masking values. This is the token used when training this model with masked language
modeling. This is the token which the model will try to predict.`,name:"mask_token"},{anchor:"transformers.NllbTokenizer.tokenizer_file",description:`<strong>tokenizer_file</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The path to a tokenizer file to use instead of the vocab file.`,name:"tokenizer_file"},{anchor:"transformers.NllbTokenizer.src_lang",description:`<strong>src_lang</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The language to use as source language for translation.`,name:"src_lang"},{anchor:"transformers.NllbTokenizer.tgt_lang",description:`<strong>tgt_lang</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The language to use as target language for translation.`,name:"tgt_lang"},{anchor:"transformers.NllbTokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>Dict[str, str]</code>) &#x2014;
Additional keyword arguments to pass to the model initialization.`,name:"sp_model_kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/nllb/tokenization_nllb.py#L49"}}),re=new Il({props:{anchor:"transformers.NllbTokenizer.example",$$slots:{default:[Wl]},$$scope:{ctx:tt}}}),xe=new Qe({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.NllbTokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.NllbTokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.NllbTokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/nllb/tokenization_nllb.py#L265",returnDescription:`
<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),Ze=new Ke({}),Be=new Qe({props:{name:"class transformers.NllbTokenizerFast",anchor:"transformers.NllbTokenizerFast",parameters:[{name:"vocab_file",val:" = None"},{name:"tokenizer_file",val:" = None"},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"sep_token",val:" = '</s>'"},{name:"cls_token",val:" = '<s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"mask_token",val:" = '<mask>'"},{name:"src_lang",val:" = None"},{name:"tgt_lang",val:" = None"},{name:"additional_special_tokens",val:" = None"},{name:"legacy_behaviour",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.NllbTokenizerFast.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to the vocabulary file.`,name:"vocab_file"},{anchor:"transformers.NllbTokenizerFast.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the beginning of
sequence. The token used is the <code>cls_token</code>.</p>

					</div>`,name:"bos_token"},{anchor:"transformers.NllbTokenizerFast.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the end of sequence.
The token used is the <code>sep_token</code>.</p>

					</div>`,name:"eos_token"},{anchor:"transformers.NllbTokenizerFast.sep_token",description:`<strong>sep_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
sequence classification or for a text and a question for question answering. It is also used as the last
token of a sequence built with special tokens.`,name:"sep_token"},{anchor:"transformers.NllbTokenizerFast.cls_token",description:`<strong>cls_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The classifier token which is used when doing sequence classification (classification of the whole sequence
instead of per-token classification). It is the first token of the sequence when built with special tokens.`,name:"cls_token"},{anchor:"transformers.NllbTokenizerFast.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.NllbTokenizerFast.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;pad&gt;&quot;</code>) &#x2014;
The token used for padding, for example when batching sequences of different lengths.`,name:"pad_token"},{anchor:"transformers.NllbTokenizerFast.mask_token",description:`<strong>mask_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;mask&gt;&quot;</code>) &#x2014;
The token used for masking values. This is the token used when training this model with masked language
modeling. This is the token which the model will try to predict.`,name:"mask_token"},{anchor:"transformers.NllbTokenizerFast.tokenizer_file",description:`<strong>tokenizer_file</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The path to a tokenizer file to use instead of the vocab file.`,name:"tokenizer_file"},{anchor:"transformers.NllbTokenizerFast.src_lang",description:`<strong>src_lang</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The language to use as source language for translation.`,name:"src_lang"},{anchor:"transformers.NllbTokenizerFast.tgt_lang",description:`<strong>tgt_lang</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The language to use as target language for translation.`,name:"tgt_lang"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/nllb/tokenization_nllb_fast.py#L61"}}),ce=new Il({props:{anchor:"transformers.NllbTokenizerFast.example",$$slots:{default:[Cl]},$$scope:{ctx:tt}}}),Se=new Qe({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.NllbTokenizerFast.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.NllbTokenizerFast.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.NllbTokenizerFast.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/nllb/tokenization_nllb_fast.py#L210",returnDescription:`
<p>list of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),Ae=new Qe({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.NllbTokenizerFast.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.NllbTokenizerFast.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.NllbTokenizerFast.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/nllb/tokenization_nllb_fast.py#L239",returnDescription:`
<p>List of zeros.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),Pe=new Qe({props:{name:"set_src_lang_special_tokens",anchor:"transformers.NllbTokenizerFast.set_src_lang_special_tokens",parameters:[{name:"src_lang",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/nllb/tokenization_nllb_fast.py#L294"}}),Ye=new Qe({props:{name:"set_tgt_lang_special_tokens",anchor:"transformers.NllbTokenizerFast.set_tgt_lang_special_tokens",parameters:[{name:"lang",val:": str"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/nllb/tokenization_nllb_fast.py#L317"}}),{c(){m=s("meta"),z=c(),w=s("h1"),u=s("a"),T=s("span"),g(f.$$.fragment),M=c(),Jt=s("span"),js=r("NLLB"),Na=c(),fe=s("p"),jt=s("strong"),Ns=r("DISCLAIMER:"),zs=r(" The default behaviour for the tokenizer has recently been fixed (and thus changed)!"),za=c(),H=s("p"),$s=r("The previous version adds "),Nt=s("code"),Us=r("[self.eos_token_id, self.cur_lang_code]"),xs=r(" at the end of the token sequence for both target and source tokenization. This is wrong as the NLLB paper mentions (page 48, 6.1.1. Model Architecture) :"),$a=c(),at=s("p"),zt=s("em"),qs=r(`Note that we prefix the source sequence with the source language, as opposed to the target
language as previously done in several works (Arivazhagan et al., 2019; Johnson et al.,
2017). This is primarily because we prioritize optimizing zero-shot performance of our
model on any pair of 200 languages at a minor cost to supervised performance.`),Ua=c(),st=s("p"),Gs=r("Previous behaviour:"),xa=c(),g(ue.$$.fragment),qa=c(),nt=s("p"),Is=r("New behaviour"),Ga=c(),g(me.$$.fragment),Ia=c(),ot=s("p"),Ls=r("Enabling the old behaviour can be done as follows:"),La=c(),g(ge.$$.fragment),Za=c(),q=s("p"),Zs=r("For more details, feel free to check the linked "),be=s("a"),Bs=r("PR"),Fs=r(" and "),ke=s("a"),Rs=r("Issue"),Ss=r("."),Ba=c(),C=s("h2"),O=s("a"),$t=s("span"),g(_e.$$.fragment),Ws=c(),Ut=s("span"),Cs=r("Overview of NLLB"),Fa=c(),Q=s("p"),Xs=r("The NLLB model was presented in "),ye=s("a"),Vs=r("No Language Left Behind: Scaling Human-Centered Machine Translation"),As=r(` by Marta R. Costa-juss\xE0, James Cross, Onur \xC7elebi,
Maha Elbayad, Kenneth Heafield, Kevin Heffernan, Elahe Kalbassi, Janice Lam, Daniel Licht, Jean Maillard, Anna Sun, Skyler Wang, Guillaume Wenzek, Al Youngblood, Bapi Akula,
Loic Barrault, Gabriel Mejia Gonzalez, Prangthip Hansanti, John Hoffman, Semarley Jarrett, Kaushik Ram Sadagopan, Dirk Rowe, Shannon Spruit, Chau Tran, Pierre Andrews,
Necip Fazil Ayan, Shruti Bhosale, Sergey Edunov, Angela Fan, Cynthia Gao, Vedanuj Goswami, Francisco Guzm\xE1n, Philipp Koehn, Alexandre Mourachko, Christophe Ropers,
Safiyyah Saleem, Holger Schwenk, and Jeff Wang.`),Ra=c(),rt=s("p"),Ps=r("The abstract of the paper is the following:"),Sa=c(),lt=s("p"),xt=s("em"),Ds=r(`Driven by the goal of eradicating language barriers on a global scale, machine translation has solidified itself as a key focus of artificial intelligence research today.
However, such efforts have coalesced around a small subset of languages, leaving behind the vast majority of mostly low-resource languages. What does it take to break the
200 language barrier while ensuring safe, high quality results, all while keeping ethical considerations in mind? In No Language Left Behind, we took on this challenge by
first contextualizing the need for low-resource language translation support through exploratory interviews with native speakers. Then, we created datasets and models aimed
at narrowing the performance gap between low and high-resource languages. More specifically, we developed a conditional compute model based on Sparsely Gated Mixture of
Experts that is trained on data obtained with novel and effective data mining techniques tailored for low-resource languages. We propose multiple architectural and training
improvements to counteract overfitting while training on thousands of tasks. Critically, we evaluated the performance of over 40,000 different translation directions using
a human-translated benchmark, Flores-200, and combined human evaluation with a novel toxicity benchmark covering all languages in Flores-200 to assess translation safety.
Our model achieves an improvement of 44% BLEU relative to the previous state-of-the-art, laying important groundwork towards realizing a universal translation system.`),Wa=c(),it=s("p"),Ys=r("This implementation contains the dense models available on release."),Ca=c(),ct=s("p"),dt=s("strong"),Hs=r("The sparse model NLLB-MoE (Mixture of Expert) is now available! More details "),pt=s("a"),Os=r("here"),Xa=c(),G=s("p"),Qs=r("This model was contributed by "),ve=s("a"),Ks=r("Lysandre"),en=r(". The authors\u2019 code can be found "),we=s("a"),tn=r("here"),an=r("."),Va=c(),X=s("h2"),K=s("a"),qt=s("span"),g(Me.$$.fragment),sn=c(),Gt=s("span"),nn=r("Generating with NLLB"),Aa=c(),I=s("p"),on=r("While generating the target text set the "),It=s("code"),rn=r("forced_bos_token_id"),ln=r(` to the target language id. The following
example shows how to translate English to French using the `),Lt=s("em"),cn=r("facebook/nllb-200-distilled-600M"),dn=r(" model."),Pa=c(),L=s("p"),pn=r("Note that we\u2019re using the BCP-47 code for French "),Zt=s("code"),hn=r("fra_Latn"),fn=r(". See "),Te=s("a"),un=r("here"),mn=r(`
for the list of all BCP-47 in the Flores 200 dataset.`),Da=c(),g(Ee.$$.fragment),Ya=c(),V=s("h3"),ee=s("a"),Bt=s("span"),g(Je.$$.fragment),gn=c(),Ft=s("span"),bn=r("Generating from any other language than English"),Ha=c(),Z=s("p"),kn=r("English ("),Rt=s("code"),_n=r("eng_Latn"),yn=r(`) is set as the default language from which to translate. In order to specify that you\u2019d like to translate from a different language,
you should specify the BCP-47 code in the `),St=s("code"),vn=r("src_lang"),wn=r(" keyword argument of the tokenizer initialization."),Oa=c(),ht=s("p"),Mn=r("See example below for a translation from romanian to german:"),Qa=c(),g(je.$$.fragment),Ka=c(),A=s("h2"),te=s("a"),Wt=s("span"),g(Ne.$$.fragment),Tn=c(),Ct=s("span"),En=r("Documentation resources"),es=c(),ae=s("ul"),Xt=s("li"),ft=s("a"),Jn=r("Translation task guide"),jn=c(),Vt=s("li"),ut=s("a"),Nn=r("Summarization task guide"),ts=c(),P=s("h2"),se=s("a"),At=s("span"),g(ze.$$.fragment),zn=c(),Pt=s("span"),$n=r("NllbTokenizer"),as=c(),E=s("div"),g($e.$$.fragment),Un=c(),Dt=s("p"),xn=r("Construct an NLLB tokenizer."),qn=c(),U=s("p"),Gn=r("Adapted from "),mt=s("a"),In=r("RobertaTokenizer"),Ln=r(" and "),gt=s("a"),Zn=r("XLNetTokenizer"),Bn=r(`. Based on
`),Ue=s("a"),Fn=r("SentencePiece"),Rn=r("."),Sn=c(),ne=s("p"),Wn=r("The tokenization method is "),Yt=s("code"),Cn=r("<tokens> <eos> <language code>"),Xn=r(" for source language documents, and `"),Ht=s("language"),Vn=c(),Ot=s("tokens"),oe=s("eos"),An=r("` for target language documents.\n"),g(re.$$.fragment),Pn=c(),$=s("div"),g(xe.$$.fragment),Dn=c(),qe=s("p"),Yn=r(`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. An NLLB sequence has the following format, where `),Qt=s("code"),Hn=r("X"),On=r(" represents the sequence:"),Qn=c(),Ge=s("ul"),Ie=s("li"),Kt=s("code"),Kn=r("input_ids"),eo=r(" (for encoder) "),ea=s("code"),to=r("X [eos, src_lang_code]"),ao=c(),Le=s("li"),ta=s("code"),so=r("decoder_input_ids"),no=r(": (for decoder) "),aa=s("code"),oo=r("X [eos, tgt_lang_code]"),ro=c(),sa=s("p"),lo=r(`BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
separator.`),ss=c(),D=s("h2"),le=s("a"),na=s("span"),g(Ze.$$.fragment),io=c(),oa=s("span"),co=r("NllbTokenizerFast"),ns=c(),J=s("div"),g(Be.$$.fragment),po=c(),Y=s("p"),ho=r("Construct a \u201Cfast\u201D NLLB tokenizer (backed by HuggingFace\u2019s "),ra=s("em"),fo=r("tokenizers"),uo=r(` library). Based on
`),Fe=s("a"),mo=r("BPE"),go=r("."),bo=c(),Re=s("p"),ko=r("This tokenizer inherits from "),bt=s("a"),_o=r("PreTrainedTokenizerFast"),yo=r(` which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.`),vo=c(),ie=s("p"),wo=r("The tokenization method is "),la=s("code"),Mo=r("<tokens> <eos> <language code>"),To=r(" for source language documents, and `"),ia=s("language"),Eo=c(),ca=s("tokens"),j=s("eos"),Jo=r("` for target language documents.\n"),g(ce.$$.fragment),jo=c(),N=s("div"),g(Se.$$.fragment),No=c(),da=s("p"),zo=r(`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. The special tokens depend on calling set_lang.`),$o=c(),We=s("p"),Uo=r("An NLLB sequence has the following format, where "),pa=s("code"),xo=r("X"),qo=r(" represents the sequence:"),Go=c(),Ce=s("ul"),Xe=s("li"),ha=s("code"),Io=r("input_ids"),Lo=r(" (for encoder) "),fa=s("code"),Zo=r("X [eos, src_lang_code]"),Bo=c(),Ve=s("li"),ua=s("code"),Fo=r("decoder_input_ids"),Ro=r(": (for decoder) "),ma=s("code"),So=r("X [eos, tgt_lang_code]"),Wo=c(),ga=s("p"),Co=r(`BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
separator.`),Xo=c(),de=s("div"),g(Ae.$$.fragment),Vo=c(),ba=s("p"),Ao=r(`Create a mask from the two sequences passed to be used in a sequence-pair classification task. nllb does not
make use of token type ids, therefore a list of zeros is returned.`),Po=c(),B=s("div"),g(Pe.$$.fragment),Do=c(),ka=s("p"),Yo=r("Reset the special tokens to the source lang setting."),Ho=c(),De=s("ul"),_a=s("li"),Oo=r("In legacy mode: No prefix and suffix=[eos, src_lang_code]."),Qo=c(),ya=s("li"),Ko=r("In default mode: Prefix=[src_lang_code], suffix = [eos]"),er=c(),F=s("div"),g(Ye.$$.fragment),tr=c(),va=s("p"),ar=r("Reset the special tokens to the target lang setting."),sr=c(),He=s("ul"),wa=s("li"),nr=r("In legacy mode: No prefix and suffix=[eos, tgt_lang_code]."),or=c(),Ma=s("li"),rr=r("In default mode: Prefix=[tgt_lang_code], suffix = [eos]"),this.h()},l(e){const i=Rl('[data-svelte="svelte-1phssyn"]',document.head);m=n(i,"META",{name:!0,content:!0}),i.forEach(a),z=d(e),w=n(e,"H1",{class:!0});var Oe=o(w);u=n(Oe,"A",{id:!0,class:!0,href:!0});var Ta=o(u);T=n(Ta,"SPAN",{});var cr=o(T);b(f.$$.fragment,cr),cr.forEach(a),Ta.forEach(a),M=d(Oe),Jt=n(Oe,"SPAN",{});var dr=o(Jt);js=l(dr,"NLLB"),dr.forEach(a),Oe.forEach(a),Na=d(e),fe=n(e,"P",{});var lr=o(fe);jt=n(lr,"STRONG",{});var pr=o(jt);Ns=l(pr,"DISCLAIMER:"),pr.forEach(a),zs=l(lr," The default behaviour for the tokenizer has recently been fixed (and thus changed)!"),lr.forEach(a),za=d(e),H=n(e,"P",{});var rs=o(H);$s=l(rs,"The previous version adds "),Nt=n(rs,"CODE",{});var hr=o(Nt);Us=l(hr,"[self.eos_token_id, self.cur_lang_code]"),hr.forEach(a),xs=l(rs," at the end of the token sequence for both target and source tokenization. This is wrong as the NLLB paper mentions (page 48, 6.1.1. Model Architecture) :"),rs.forEach(a),$a=d(e),at=n(e,"P",{});var fr=o(at);zt=n(fr,"EM",{});var ur=o(zt);qs=l(ur,`Note that we prefix the source sequence with the source language, as opposed to the target
language as previously done in several works (Arivazhagan et al., 2019; Johnson et al.,
2017). This is primarily because we prioritize optimizing zero-shot performance of our
model on any pair of 200 languages at a minor cost to supervised performance.`),ur.forEach(a),fr.forEach(a),Ua=d(e),st=n(e,"P",{});var mr=o(st);Gs=l(mr,"Previous behaviour:"),mr.forEach(a),xa=d(e),b(ue.$$.fragment,e),qa=d(e),nt=n(e,"P",{});var gr=o(nt);Is=l(gr,"New behaviour"),gr.forEach(a),Ga=d(e),b(me.$$.fragment,e),Ia=d(e),ot=n(e,"P",{});var br=o(ot);Ls=l(br,"Enabling the old behaviour can be done as follows:"),br.forEach(a),La=d(e),b(ge.$$.fragment,e),Za=d(e),q=n(e,"P",{});var kt=o(q);Zs=l(kt,"For more details, feel free to check the linked "),be=n(kt,"A",{href:!0,rel:!0});var kr=o(be);Bs=l(kr,"PR"),kr.forEach(a),Fs=l(kt," and "),ke=n(kt,"A",{href:!0,rel:!0});var _r=o(ke);Rs=l(_r,"Issue"),_r.forEach(a),Ss=l(kt,"."),kt.forEach(a),Ba=d(e),C=n(e,"H2",{class:!0});var ls=o(C);O=n(ls,"A",{id:!0,class:!0,href:!0});var yr=o(O);$t=n(yr,"SPAN",{});var vr=o($t);b(_e.$$.fragment,vr),vr.forEach(a),yr.forEach(a),Ws=d(ls),Ut=n(ls,"SPAN",{});var wr=o(Ut);Cs=l(wr,"Overview of NLLB"),wr.forEach(a),ls.forEach(a),Fa=d(e),Q=n(e,"P",{});var is=o(Q);Xs=l(is,"The NLLB model was presented in "),ye=n(is,"A",{href:!0,rel:!0});var Mr=o(ye);Vs=l(Mr,"No Language Left Behind: Scaling Human-Centered Machine Translation"),Mr.forEach(a),As=l(is,` by Marta R. Costa-juss\xE0, James Cross, Onur \xC7elebi,
Maha Elbayad, Kenneth Heafield, Kevin Heffernan, Elahe Kalbassi, Janice Lam, Daniel Licht, Jean Maillard, Anna Sun, Skyler Wang, Guillaume Wenzek, Al Youngblood, Bapi Akula,
Loic Barrault, Gabriel Mejia Gonzalez, Prangthip Hansanti, John Hoffman, Semarley Jarrett, Kaushik Ram Sadagopan, Dirk Rowe, Shannon Spruit, Chau Tran, Pierre Andrews,
Necip Fazil Ayan, Shruti Bhosale, Sergey Edunov, Angela Fan, Cynthia Gao, Vedanuj Goswami, Francisco Guzm\xE1n, Philipp Koehn, Alexandre Mourachko, Christophe Ropers,
Safiyyah Saleem, Holger Schwenk, and Jeff Wang.`),is.forEach(a),Ra=d(e),rt=n(e,"P",{});var Tr=o(rt);Ps=l(Tr,"The abstract of the paper is the following:"),Tr.forEach(a),Sa=d(e),lt=n(e,"P",{});var Er=o(lt);xt=n(Er,"EM",{});var Jr=o(xt);Ds=l(Jr,`Driven by the goal of eradicating language barriers on a global scale, machine translation has solidified itself as a key focus of artificial intelligence research today.
However, such efforts have coalesced around a small subset of languages, leaving behind the vast majority of mostly low-resource languages. What does it take to break the
200 language barrier while ensuring safe, high quality results, all while keeping ethical considerations in mind? In No Language Left Behind, we took on this challenge by
first contextualizing the need for low-resource language translation support through exploratory interviews with native speakers. Then, we created datasets and models aimed
at narrowing the performance gap between low and high-resource languages. More specifically, we developed a conditional compute model based on Sparsely Gated Mixture of
Experts that is trained on data obtained with novel and effective data mining techniques tailored for low-resource languages. We propose multiple architectural and training
improvements to counteract overfitting while training on thousands of tasks. Critically, we evaluated the performance of over 40,000 different translation directions using
a human-translated benchmark, Flores-200, and combined human evaluation with a novel toxicity benchmark covering all languages in Flores-200 to assess translation safety.
Our model achieves an improvement of 44% BLEU relative to the previous state-of-the-art, laying important groundwork towards realizing a universal translation system.`),Jr.forEach(a),Er.forEach(a),Wa=d(e),it=n(e,"P",{});var jr=o(it);Ys=l(jr,"This implementation contains the dense models available on release."),jr.forEach(a),Ca=d(e),ct=n(e,"P",{});var Nr=o(ct);dt=n(Nr,"STRONG",{});var ir=o(dt);Hs=l(ir,"The sparse model NLLB-MoE (Mixture of Expert) is now available! More details "),pt=n(ir,"A",{href:!0});var zr=o(pt);Os=l(zr,"here"),zr.forEach(a),ir.forEach(a),Nr.forEach(a),Xa=d(e),G=n(e,"P",{});var _t=o(G);Qs=l(_t,"This model was contributed by "),ve=n(_t,"A",{href:!0,rel:!0});var $r=o(ve);Ks=l($r,"Lysandre"),$r.forEach(a),en=l(_t,". The authors\u2019 code can be found "),we=n(_t,"A",{href:!0,rel:!0});var Ur=o(we);tn=l(Ur,"here"),Ur.forEach(a),an=l(_t,"."),_t.forEach(a),Va=d(e),X=n(e,"H2",{class:!0});var cs=o(X);K=n(cs,"A",{id:!0,class:!0,href:!0});var xr=o(K);qt=n(xr,"SPAN",{});var qr=o(qt);b(Me.$$.fragment,qr),qr.forEach(a),xr.forEach(a),sn=d(cs),Gt=n(cs,"SPAN",{});var Gr=o(Gt);nn=l(Gr,"Generating with NLLB"),Gr.forEach(a),cs.forEach(a),Aa=d(e),I=n(e,"P",{});var yt=o(I);on=l(yt,"While generating the target text set the "),It=n(yt,"CODE",{});var Ir=o(It);rn=l(Ir,"forced_bos_token_id"),Ir.forEach(a),ln=l(yt,` to the target language id. The following
example shows how to translate English to French using the `),Lt=n(yt,"EM",{});var Lr=o(Lt);cn=l(Lr,"facebook/nllb-200-distilled-600M"),Lr.forEach(a),dn=l(yt," model."),yt.forEach(a),Pa=d(e),L=n(e,"P",{});var vt=o(L);pn=l(vt,"Note that we\u2019re using the BCP-47 code for French "),Zt=n(vt,"CODE",{});var Zr=o(Zt);hn=l(Zr,"fra_Latn"),Zr.forEach(a),fn=l(vt,". See "),Te=n(vt,"A",{href:!0,rel:!0});var Br=o(Te);un=l(Br,"here"),Br.forEach(a),mn=l(vt,`
for the list of all BCP-47 in the Flores 200 dataset.`),vt.forEach(a),Da=d(e),b(Ee.$$.fragment,e),Ya=d(e),V=n(e,"H3",{class:!0});var ds=o(V);ee=n(ds,"A",{id:!0,class:!0,href:!0});var Fr=o(ee);Bt=n(Fr,"SPAN",{});var Rr=o(Bt);b(Je.$$.fragment,Rr),Rr.forEach(a),Fr.forEach(a),gn=d(ds),Ft=n(ds,"SPAN",{});var Sr=o(Ft);bn=l(Sr,"Generating from any other language than English"),Sr.forEach(a),ds.forEach(a),Ha=d(e),Z=n(e,"P",{});var wt=o(Z);kn=l(wt,"English ("),Rt=n(wt,"CODE",{});var Wr=o(Rt);_n=l(Wr,"eng_Latn"),Wr.forEach(a),yn=l(wt,`) is set as the default language from which to translate. In order to specify that you\u2019d like to translate from a different language,
you should specify the BCP-47 code in the `),St=n(wt,"CODE",{});var Cr=o(St);vn=l(Cr,"src_lang"),Cr.forEach(a),wn=l(wt," keyword argument of the tokenizer initialization."),wt.forEach(a),Oa=d(e),ht=n(e,"P",{});var Xr=o(ht);Mn=l(Xr,"See example below for a translation from romanian to german:"),Xr.forEach(a),Qa=d(e),b(je.$$.fragment,e),Ka=d(e),A=n(e,"H2",{class:!0});var ps=o(A);te=n(ps,"A",{id:!0,class:!0,href:!0});var Vr=o(te);Wt=n(Vr,"SPAN",{});var Ar=o(Wt);b(Ne.$$.fragment,Ar),Ar.forEach(a),Vr.forEach(a),Tn=d(ps),Ct=n(ps,"SPAN",{});var Pr=o(Ct);En=l(Pr,"Documentation resources"),Pr.forEach(a),ps.forEach(a),es=d(e),ae=n(e,"UL",{});var hs=o(ae);Xt=n(hs,"LI",{});var Dr=o(Xt);ft=n(Dr,"A",{href:!0});var Yr=o(ft);Jn=l(Yr,"Translation task guide"),Yr.forEach(a),Dr.forEach(a),jn=d(hs),Vt=n(hs,"LI",{});var Hr=o(Vt);ut=n(Hr,"A",{href:!0});var Or=o(ut);Nn=l(Or,"Summarization task guide"),Or.forEach(a),Hr.forEach(a),hs.forEach(a),ts=d(e),P=n(e,"H2",{class:!0});var fs=o(P);se=n(fs,"A",{id:!0,class:!0,href:!0});var Qr=o(se);At=n(Qr,"SPAN",{});var Kr=o(At);b(ze.$$.fragment,Kr),Kr.forEach(a),Qr.forEach(a),zn=d(fs),Pt=n(fs,"SPAN",{});var el=o(Pt);$n=l(el,"NllbTokenizer"),el.forEach(a),fs.forEach(a),as=d(e),E=n(e,"DIV",{class:!0});var R=o(E);b($e.$$.fragment,R),Un=d(R),Dt=n(R,"P",{});var tl=o(Dt);xn=l(tl,"Construct an NLLB tokenizer."),tl.forEach(a),qn=d(R),U=n(R,"P",{});var pe=o(U);Gn=l(pe,"Adapted from "),mt=n(pe,"A",{href:!0});var al=o(mt);In=l(al,"RobertaTokenizer"),al.forEach(a),Ln=l(pe," and "),gt=n(pe,"A",{href:!0});var sl=o(gt);Zn=l(sl,"XLNetTokenizer"),sl.forEach(a),Bn=l(pe,`. Based on
`),Ue=n(pe,"A",{href:!0,rel:!0});var nl=o(Ue);Fn=l(nl,"SentencePiece"),nl.forEach(a),Rn=l(pe,"."),pe.forEach(a),Sn=d(R),ne=n(R,"P",{});var Ea=o(ne);Wn=l(Ea,"The tokenization method is "),Yt=n(Ea,"CODE",{});var ol=o(Yt);Cn=l(ol,"<tokens> <eos> <language code>"),ol.forEach(a),Xn=l(Ea," for source language documents, and `"),Ht=n(Ea,"LANGUAGE",{code:!0}),o(Ht).forEach(a),Ea.forEach(a),Vn=d(R),Ot=n(R,"TOKENS",{});var rl=o(Ot);oe=n(rl,"EOS",{});var Ja=o(oe);An=l(Ja,"` for target language documents.\n"),b(re.$$.fragment,Ja),Pn=d(Ja),$=n(Ja,"DIV",{class:!0});var he=o($);b(xe.$$.fragment,he),Dn=d(he),qe=n(he,"P",{});var us=o(qe);Yn=l(us,`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. An NLLB sequence has the following format, where `),Qt=n(us,"CODE",{});var ll=o(Qt);Hn=l(ll,"X"),ll.forEach(a),On=l(us," represents the sequence:"),us.forEach(a),Qn=d(he),Ge=n(he,"UL",{});var ms=o(Ge);Ie=n(ms,"LI",{});var gs=o(Ie);Kt=n(gs,"CODE",{});var il=o(Kt);Kn=l(il,"input_ids"),il.forEach(a),eo=l(gs," (for encoder) "),ea=n(gs,"CODE",{});var cl=o(ea);to=l(cl,"X [eos, src_lang_code]"),cl.forEach(a),gs.forEach(a),ao=d(ms),Le=n(ms,"LI",{});var bs=o(Le);ta=n(bs,"CODE",{});var dl=o(ta);so=l(dl,"decoder_input_ids"),dl.forEach(a),no=l(bs,": (for decoder) "),aa=n(bs,"CODE",{});var pl=o(aa);oo=l(pl,"X [eos, tgt_lang_code]"),pl.forEach(a),bs.forEach(a),ms.forEach(a),ro=d(he),sa=n(he,"P",{});var hl=o(sa);lo=l(hl,`BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
separator.`),hl.forEach(a),he.forEach(a),Ja.forEach(a),rl.forEach(a),R.forEach(a),ss=d(e),D=n(e,"H2",{class:!0});var ks=o(D);le=n(ks,"A",{id:!0,class:!0,href:!0});var fl=o(le);na=n(fl,"SPAN",{});var ul=o(na);b(Ze.$$.fragment,ul),ul.forEach(a),fl.forEach(a),io=d(ks),oa=n(ks,"SPAN",{});var ml=o(oa);co=l(ml,"NllbTokenizerFast"),ml.forEach(a),ks.forEach(a),ns=d(e),J=n(e,"DIV",{class:!0});var S=o(J);b(Be.$$.fragment,S),po=d(S),Y=n(S,"P",{});var Mt=o(Y);ho=l(Mt,"Construct a \u201Cfast\u201D NLLB tokenizer (backed by HuggingFace\u2019s "),ra=n(Mt,"EM",{});var gl=o(ra);fo=l(gl,"tokenizers"),gl.forEach(a),uo=l(Mt,` library). Based on
`),Fe=n(Mt,"A",{href:!0,rel:!0});var bl=o(Fe);mo=l(bl,"BPE"),bl.forEach(a),go=l(Mt,"."),Mt.forEach(a),bo=d(S),Re=n(S,"P",{});var _s=o(Re);ko=l(_s,"This tokenizer inherits from "),bt=n(_s,"A",{href:!0});var kl=o(bt);_o=l(kl,"PreTrainedTokenizerFast"),kl.forEach(a),yo=l(_s,` which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.`),_s.forEach(a),vo=d(S),ie=n(S,"P",{});var ja=o(ie);wo=l(ja,"The tokenization method is "),la=n(ja,"CODE",{});var _l=o(la);Mo=l(_l,"<tokens> <eos> <language code>"),_l.forEach(a),To=l(ja," for source language documents, and `"),ia=n(ja,"LANGUAGE",{code:!0}),o(ia).forEach(a),ja.forEach(a),Eo=d(S),ca=n(S,"TOKENS",{});var yl=o(ca);j=n(yl,"EOS",{});var x=o(j);Jo=l(x,"` for target language documents.\n"),b(ce.$$.fragment,x),jo=d(x),N=n(x,"DIV",{class:!0});var W=o(N);b(Se.$$.fragment,W),No=d(W),da=n(W,"P",{});var vl=o(da);zo=l(vl,`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. The special tokens depend on calling set_lang.`),vl.forEach(a),$o=d(W),We=n(W,"P",{});var ys=o(We);Uo=l(ys,"An NLLB sequence has the following format, where "),pa=n(ys,"CODE",{});var wl=o(pa);xo=l(wl,"X"),wl.forEach(a),qo=l(ys," represents the sequence:"),ys.forEach(a),Go=d(W),Ce=n(W,"UL",{});var vs=o(Ce);Xe=n(vs,"LI",{});var ws=o(Xe);ha=n(ws,"CODE",{});var Ml=o(ha);Io=l(Ml,"input_ids"),Ml.forEach(a),Lo=l(ws," (for encoder) "),fa=n(ws,"CODE",{});var Tl=o(fa);Zo=l(Tl,"X [eos, src_lang_code]"),Tl.forEach(a),ws.forEach(a),Bo=d(vs),Ve=n(vs,"LI",{});var Ms=o(Ve);ua=n(Ms,"CODE",{});var El=o(ua);Fo=l(El,"decoder_input_ids"),El.forEach(a),Ro=l(Ms,": (for decoder) "),ma=n(Ms,"CODE",{});var Jl=o(ma);So=l(Jl,"X [eos, tgt_lang_code]"),Jl.forEach(a),Ms.forEach(a),vs.forEach(a),Wo=d(W),ga=n(W,"P",{});var jl=o(ga);Co=l(jl,`BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
separator.`),jl.forEach(a),W.forEach(a),Xo=d(x),de=n(x,"DIV",{class:!0});var Ts=o(de);b(Ae.$$.fragment,Ts),Vo=d(Ts),ba=n(Ts,"P",{});var Nl=o(ba);Ao=l(Nl,`Create a mask from the two sequences passed to be used in a sequence-pair classification task. nllb does not
make use of token type ids, therefore a list of zeros is returned.`),Nl.forEach(a),Ts.forEach(a),Po=d(x),B=n(x,"DIV",{class:!0});var Tt=o(B);b(Pe.$$.fragment,Tt),Do=d(Tt),ka=n(Tt,"P",{});var zl=o(ka);Yo=l(zl,"Reset the special tokens to the source lang setting."),zl.forEach(a),Ho=d(Tt),De=n(Tt,"UL",{});var Es=o(De);_a=n(Es,"LI",{});var $l=o(_a);Oo=l($l,"In legacy mode: No prefix and suffix=[eos, src_lang_code]."),$l.forEach(a),Qo=d(Es),ya=n(Es,"LI",{});var Ul=o(ya);Ko=l(Ul,"In default mode: Prefix=[src_lang_code], suffix = [eos]"),Ul.forEach(a),Es.forEach(a),Tt.forEach(a),er=d(x),F=n(x,"DIV",{class:!0});var Et=o(F);b(Ye.$$.fragment,Et),tr=d(Et),va=n(Et,"P",{});var xl=o(va);ar=l(xl,"Reset the special tokens to the target lang setting."),xl.forEach(a),sr=d(Et),He=n(Et,"UL",{});var Js=o(He);wa=n(Js,"LI",{});var ql=o(wa);nr=l(ql,"In legacy mode: No prefix and suffix=[eos, tgt_lang_code]."),ql.forEach(a),or=d(Js),Ma=n(Js,"LI",{});var Gl=o(Ma);rr=l(Gl,"In default mode: Prefix=[tgt_lang_code], suffix = [eos]"),Gl.forEach(a),Js.forEach(a),Et.forEach(a),x.forEach(a),yl.forEach(a),S.forEach(a),this.h()},h(){h(m,"name","hf:doc:metadata"),h(m,"content",JSON.stringify(Vl)),h(u,"id","transformers.NllbTokenizer"),h(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(u,"href","#transformers.NllbTokenizer"),h(w,"class","relative group"),h(be,"href","https://github.com/huggingface/transformers/pull/22313"),h(be,"rel","nofollow"),h(ke,"href","https://github.com/huggingface/transformers/issues/19943"),h(ke,"rel","nofollow"),h(O,"id","overview-of-nllb"),h(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(O,"href","#overview-of-nllb"),h(C,"class","relative group"),h(ye,"href","https://arxiv.org/abs/2207.04672"),h(ye,"rel","nofollow"),h(pt,"href","nllb-moe"),h(ve,"href","https://huggingface.co/lysandre"),h(ve,"rel","nofollow"),h(we,"href","https://github.com/facebookresearch/fairseq/tree/nllb"),h(we,"rel","nofollow"),h(K,"id","generating-with-nllb"),h(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(K,"href","#generating-with-nllb"),h(X,"class","relative group"),h(Te,"href","https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200"),h(Te,"rel","nofollow"),h(ee,"id","generating-from-any-other-language-than-english"),h(ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ee,"href","#generating-from-any-other-language-than-english"),h(V,"class","relative group"),h(te,"id","documentation-resources"),h(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(te,"href","#documentation-resources"),h(A,"class","relative group"),h(ft,"href","../tasks/translation"),h(ut,"href","../tasks/summarization"),h(se,"id","nllbtokenizer"),h(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(se,"href","#nllbtokenizer"),h(P,"class","relative group"),h(mt,"href","/docs/transformers/v4.33.0/en/model_doc/roberta#transformers.RobertaTokenizer"),h(gt,"href","/docs/transformers/v4.33.0/en/model_doc/xlnet#transformers.XLNetTokenizer"),h(Ue,"href","https://github.com/google/sentencepiece"),h(Ue,"rel","nofollow"),h(Ht,"code",""),h($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(le,"id","nllbtokenizerfast"),h(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(le,"href","#nllbtokenizerfast"),h(D,"class","relative group"),h(Fe,"href","https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models"),h(Fe,"rel","nofollow"),h(bt,"href","/docs/transformers/v4.33.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast"),h(ia,"code",""),h(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(de,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,i){t(document.head,m),p(e,z,i),p(e,w,i),t(w,u),t(u,T),k(f,T,null),t(w,M),t(w,Jt),t(Jt,js),p(e,Na,i),p(e,fe,i),t(fe,jt),t(jt,Ns),t(fe,zs),p(e,za,i),p(e,H,i),t(H,$s),t(H,Nt),t(Nt,Us),t(H,xs),p(e,$a,i),p(e,at,i),t(at,zt),t(zt,qs),p(e,Ua,i),p(e,st,i),t(st,Gs),p(e,xa,i),k(ue,e,i),p(e,qa,i),p(e,nt,i),t(nt,Is),p(e,Ga,i),k(me,e,i),p(e,Ia,i),p(e,ot,i),t(ot,Ls),p(e,La,i),k(ge,e,i),p(e,Za,i),p(e,q,i),t(q,Zs),t(q,be),t(be,Bs),t(q,Fs),t(q,ke),t(ke,Rs),t(q,Ss),p(e,Ba,i),p(e,C,i),t(C,O),t(O,$t),k(_e,$t,null),t(C,Ws),t(C,Ut),t(Ut,Cs),p(e,Fa,i),p(e,Q,i),t(Q,Xs),t(Q,ye),t(ye,Vs),t(Q,As),p(e,Ra,i),p(e,rt,i),t(rt,Ps),p(e,Sa,i),p(e,lt,i),t(lt,xt),t(xt,Ds),p(e,Wa,i),p(e,it,i),t(it,Ys),p(e,Ca,i),p(e,ct,i),t(ct,dt),t(dt,Hs),t(dt,pt),t(pt,Os),p(e,Xa,i),p(e,G,i),t(G,Qs),t(G,ve),t(ve,Ks),t(G,en),t(G,we),t(we,tn),t(G,an),p(e,Va,i),p(e,X,i),t(X,K),t(K,qt),k(Me,qt,null),t(X,sn),t(X,Gt),t(Gt,nn),p(e,Aa,i),p(e,I,i),t(I,on),t(I,It),t(It,rn),t(I,ln),t(I,Lt),t(Lt,cn),t(I,dn),p(e,Pa,i),p(e,L,i),t(L,pn),t(L,Zt),t(Zt,hn),t(L,fn),t(L,Te),t(Te,un),t(L,mn),p(e,Da,i),k(Ee,e,i),p(e,Ya,i),p(e,V,i),t(V,ee),t(ee,Bt),k(Je,Bt,null),t(V,gn),t(V,Ft),t(Ft,bn),p(e,Ha,i),p(e,Z,i),t(Z,kn),t(Z,Rt),t(Rt,_n),t(Z,yn),t(Z,St),t(St,vn),t(Z,wn),p(e,Oa,i),p(e,ht,i),t(ht,Mn),p(e,Qa,i),k(je,e,i),p(e,Ka,i),p(e,A,i),t(A,te),t(te,Wt),k(Ne,Wt,null),t(A,Tn),t(A,Ct),t(Ct,En),p(e,es,i),p(e,ae,i),t(ae,Xt),t(Xt,ft),t(ft,Jn),t(ae,jn),t(ae,Vt),t(Vt,ut),t(ut,Nn),p(e,ts,i),p(e,P,i),t(P,se),t(se,At),k(ze,At,null),t(P,zn),t(P,Pt),t(Pt,$n),p(e,as,i),p(e,E,i),k($e,E,null),t(E,Un),t(E,Dt),t(Dt,xn),t(E,qn),t(E,U),t(U,Gn),t(U,mt),t(mt,In),t(U,Ln),t(U,gt),t(gt,Zn),t(U,Bn),t(U,Ue),t(Ue,Fn),t(U,Rn),t(E,Sn),t(E,ne),t(ne,Wn),t(ne,Yt),t(Yt,Cn),t(ne,Xn),t(ne,Ht),t(E,Vn),t(E,Ot),t(Ot,oe),t(oe,An),k(re,oe,null),t(oe,Pn),t(oe,$),k(xe,$,null),t($,Dn),t($,qe),t(qe,Yn),t(qe,Qt),t(Qt,Hn),t(qe,On),t($,Qn),t($,Ge),t(Ge,Ie),t(Ie,Kt),t(Kt,Kn),t(Ie,eo),t(Ie,ea),t(ea,to),t(Ge,ao),t(Ge,Le),t(Le,ta),t(ta,so),t(Le,no),t(Le,aa),t(aa,oo),t($,ro),t($,sa),t(sa,lo),p(e,ss,i),p(e,D,i),t(D,le),t(le,na),k(Ze,na,null),t(D,io),t(D,oa),t(oa,co),p(e,ns,i),p(e,J,i),k(Be,J,null),t(J,po),t(J,Y),t(Y,ho),t(Y,ra),t(ra,fo),t(Y,uo),t(Y,Fe),t(Fe,mo),t(Y,go),t(J,bo),t(J,Re),t(Re,ko),t(Re,bt),t(bt,_o),t(Re,yo),t(J,vo),t(J,ie),t(ie,wo),t(ie,la),t(la,Mo),t(ie,To),t(ie,ia),t(J,Eo),t(J,ca),t(ca,j),t(j,Jo),k(ce,j,null),t(j,jo),t(j,N),k(Se,N,null),t(N,No),t(N,da),t(da,zo),t(N,$o),t(N,We),t(We,Uo),t(We,pa),t(pa,xo),t(We,qo),t(N,Go),t(N,Ce),t(Ce,Xe),t(Xe,ha),t(ha,Io),t(Xe,Lo),t(Xe,fa),t(fa,Zo),t(Ce,Bo),t(Ce,Ve),t(Ve,ua),t(ua,Fo),t(Ve,Ro),t(Ve,ma),t(ma,So),t(N,Wo),t(N,ga),t(ga,Co),t(j,Xo),t(j,de),k(Ae,de,null),t(de,Vo),t(de,ba),t(ba,Ao),t(j,Po),t(j,B),k(Pe,B,null),t(B,Do),t(B,ka),t(ka,Yo),t(B,Ho),t(B,De),t(De,_a),t(_a,Oo),t(De,Qo),t(De,ya),t(ya,Ko),t(j,er),t(j,F),k(Ye,F,null),t(F,tr),t(F,va),t(va,ar),t(F,sr),t(F,He),t(He,wa),t(wa,nr),t(He,or),t(He,Ma),t(Ma,rr),os=!0},p(e,[i]){const Oe={};i&2&&(Oe.$$scope={dirty:i,ctx:e}),re.$set(Oe);const Ta={};i&2&&(Ta.$$scope={dirty:i,ctx:e}),ce.$set(Ta)},i(e){os||(_(f.$$.fragment,e),_(ue.$$.fragment,e),_(me.$$.fragment,e),_(ge.$$.fragment,e),_(_e.$$.fragment,e),_(Me.$$.fragment,e),_(Ee.$$.fragment,e),_(Je.$$.fragment,e),_(je.$$.fragment,e),_(Ne.$$.fragment,e),_(ze.$$.fragment,e),_($e.$$.fragment,e),_(re.$$.fragment,e),_(xe.$$.fragment,e),_(Ze.$$.fragment,e),_(Be.$$.fragment,e),_(ce.$$.fragment,e),_(Se.$$.fragment,e),_(Ae.$$.fragment,e),_(Pe.$$.fragment,e),_(Ye.$$.fragment,e),os=!0)},o(e){y(f.$$.fragment,e),y(ue.$$.fragment,e),y(me.$$.fragment,e),y(ge.$$.fragment,e),y(_e.$$.fragment,e),y(Me.$$.fragment,e),y(Ee.$$.fragment,e),y(Je.$$.fragment,e),y(je.$$.fragment,e),y(Ne.$$.fragment,e),y(ze.$$.fragment,e),y($e.$$.fragment,e),y(re.$$.fragment,e),y(xe.$$.fragment,e),y(Ze.$$.fragment,e),y(Be.$$.fragment,e),y(ce.$$.fragment,e),y(Se.$$.fragment,e),y(Ae.$$.fragment,e),y(Pe.$$.fragment,e),y(Ye.$$.fragment,e),os=!1},d(e){a(m),e&&a(z),e&&a(w),v(f),e&&a(Na),e&&a(fe),e&&a(za),e&&a(H),e&&a($a),e&&a(at),e&&a(Ua),e&&a(st),e&&a(xa),v(ue,e),e&&a(qa),e&&a(nt),e&&a(Ga),v(me,e),e&&a(Ia),e&&a(ot),e&&a(La),v(ge,e),e&&a(Za),e&&a(q),e&&a(Ba),e&&a(C),v(_e),e&&a(Fa),e&&a(Q),e&&a(Ra),e&&a(rt),e&&a(Sa),e&&a(lt),e&&a(Wa),e&&a(it),e&&a(Ca),e&&a(ct),e&&a(Xa),e&&a(G),e&&a(Va),e&&a(X),v(Me),e&&a(Aa),e&&a(I),e&&a(Pa),e&&a(L),e&&a(Da),v(Ee,e),e&&a(Ya),e&&a(V),v(Je),e&&a(Ha),e&&a(Z),e&&a(Oa),e&&a(ht),e&&a(Qa),v(je,e),e&&a(Ka),e&&a(A),v(Ne),e&&a(es),e&&a(ae),e&&a(ts),e&&a(P),v(ze),e&&a(as),e&&a(E),v($e),v(re),v(xe),e&&a(ss),e&&a(D),v(Ze),e&&a(ns),e&&a(J),v(Be),v(ce),v(Se),v(Ae),v(Pe),v(Ye)}}}const Vl={local:"transformers.NllbTokenizer",title:"NLLB"};function Al(tt){return Sl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ql extends Zl{constructor(m){super();Bl(this,m,Al,Xl,Fl,{})}}export{Ql as default,Vl as metadata};
