import{S as _n,i as kn,s as Tn,e as o,k as d,w as M,t as s,M as bn,c as r,d as n,m,a,x as z,h as l,b as c,G as t,g as p,y as $,q as G,o as x,B as A,v as yn,L as Sn}from"../../chunks/vendor-hf-doc-builder.js";import{D as wn}from"../../chunks/Docstring-hf-doc-builder.js";import{C as vn}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Fe}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as En}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function Pn(Se){let u,U,_,g,k;return g=new vn({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVFN3M1Rva2VuaXplciUwQSUwQXRva2VuaXplciUyMCUzRCUyMEdQVFN3M1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyQUktU3dlZGVuJTJGZ3B0LXN3My0xMjZtJTIyKSUwQXRva2VuaXplciglMjJTdmVuc2thJTIwJUMzJUE0ciUyMGt1bCElMjIpJTVCJTIyaW5wdXRfaWRzJTIyJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSw3Tokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTSw3Tokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-126m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;Svenska \xE4r kul!&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[<span class="hljs-number">1814</span>, <span class="hljs-number">377</span>, <span class="hljs-number">3617</span>, <span class="hljs-number">63504</span>]`}}),{c(){u=o("p"),U=s("Example usage:"),_=d(),M(g.$$.fragment)},l(h){u=r(h,"P",{});var T=a(u);U=l(T,"Example usage:"),T.forEach(n),_=m(h),z(g.$$.fragment,h)},m(h,T){p(h,u,T),t(u,U),p(h,_,T),$(g,h,T),k=!0},p:Sn,i(h){k||(G(g.$$.fragment,h),k=!0)},o(h){x(g.$$.fragment,h),k=!1},d(h){h&&n(u),h&&n(_),A(g,h)}}}function Mn(Se){let u,U,_,g,k,h,T,ie,qe,Ee,S,C,ce,V,De,pe,Xe,Pe,j,He,B,Qe,Oe,Me,K,Ye,ze,ee,Ke,$e,I,et,W,tt,nt,Ge,f,ot,Z,rt,at,de,st,lt,me,it,ct,he,pt,dt,fe,mt,ht,ue,ft,ut,ge,gt,xe,te,wt,Ae,L,Ue,E,J,we,F,vt,ve,_t,Ce,b,_e,ne,kt,Tt,ke,oe,bt,yt,Te,re,St,je,P,N,be,q,Et,ye,Pt,Ie,w,D,Mt,X,zt,H,$t,Gt,xt,Q,At,ae,Ut,Ct,jt,R,It,se,O,Je;return h=new Fe({}),V=new Fe({}),L=new vn({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMkFJLVN3ZWRlbiUyRmdwdC1zdzMtMzU2bSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJBSS1Td2VkZW4lMkZncHQtc3czLTM1Nm0lMjIpJTBBJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMlRyJUMzJUE0ZCUyMCVDMyVBNHIlMjBmaW5hJTIwZiVDMyVCNnIlMjBhdHQlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUwQSUwQWdlbmVyYXRlZF90b2tlbl9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dHMlM0RpbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwJTJDJTIwZG9fc2FtcGxlJTNEVHJ1ZSklNUIwJTVEJTBBJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShnZW5lcmF0ZWRfdG9rZW5faWRzKSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-356m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-356m&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(<span class="hljs-string">&quot;Tr\xE4d \xE4r fina f\xF6r att&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_token_ids = model.generate(inputs=input_ids, max_new_tokens=<span class="hljs-number">10</span>, do_sample=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(generated_token_ids))
Tr\xE4d \xE4r fina f\xF6r att de \xE4r f\xE4rgstarka. Men ibland \xE4r det fint`}}),F=new Fe({}),q=new Fe({}),D=new wn({props:{name:"class transformers.GPTSw3Tokenizer",anchor:"transformers.GPTSw3Tokenizer",parameters:[{name:"vocab_file",val:""},{name:"do_lower_case",val:" = False"},{name:"remove_space",val:" = False"},{name:"keep_accents",val:" = False"},{name:"pad_token",val:" = None"},{name:"unk_token",val:" = None"},{name:"eos_token",val:" = None"},{name:"bos_token",val:" = None"},{name:"sp_model_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTSw3Tokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
<a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a <em>.spm</em> extension) that
contains the vocabulary necessary to instantiate a tokenizer.`,name:"vocab_file"},{anchor:"transformers.GPTSw3Tokenizer.do_lower_case",description:`<strong>do_lower_case</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to lowercase the input when tokenizing.`,name:"do_lower_case"},{anchor:"transformers.GPTSw3Tokenizer.remove_space",description:`<strong>remove_space</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to strip the text when tokenizing (removing excess spaces before and after the string).`,name:"remove_space"},{anchor:"transformers.GPTSw3Tokenizer.keep_accents",description:`<strong>keep_accents</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to keep accents when tokenizing.`,name:"keep_accents"},{anchor:"transformers.GPTSw3Tokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The beginning of sequence token that can be used for downstream task, was not seen during pretraining. If
not provided, will default to &#x2019;<s>&#x2019; or &#x2019;&lt;|endoftext|&gt;&#x2019;, depending on model size.</s>`,name:"bos_token"},{anchor:"transformers.GPTSw3Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The end of sequence token seen during pretraining. If not provided, will default to &#x2019;&lt;|endoftext|&gt;&#x2019;`,name:"eos_token"},{anchor:"transformers.GPTSw3Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead. If not provided, will default to &#x2019;<unk>&#x2018;.</unk>`,name:"unk_token"},{anchor:"transformers.GPTSw3Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The token used for padding, for example when batching sequences of different lengths. If not provided, will
default to &#x2019;<pad>&#x2019; or &#x2019;<unk>&#x2019; depending on model size.</unk></pad>`,name:"pad_token"},{anchor:"transformers.GPTSw3Tokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.GPTSw3Tokenizer.sp_model",description:`<strong>sp_model</strong> (<code>SentencePieceProcessor</code>) &#x2014;
The <em>SentencePiece</em> processor that is used for every conversion (string, tokens and IDs).`,name:"sp_model"},{anchor:"transformers.GPTSw3Tokenizer.whitespaces",description:`<strong>whitespaces</strong> (<code>set</code>) &#x2014;
The whitespaces that are replaced in the whitespace normalization in preprocessing.`,name:"whitespaces"},{anchor:"transformers.GPTSw3Tokenizer.non_printing_characters_re",description:`<strong>non_printing_characters_re</strong> (<code>Pattern</code>) &#x2014;
The compiled regular expression to remove non-printing characters in preprocessing.`,name:"non_printing_characters_re"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L45"}}),R=new En({props:{anchor:"transformers.GPTSw3Tokenizer.example",$$slots:{default:[Pn]},$$scope:{ctx:Se}}}),O=new wn({props:{name:"save_vocabulary",anchor:"transformers.GPTSw3Tokenizer.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L258"}}),{c(){u=o("meta"),U=d(),_=o("h1"),g=o("a"),k=o("span"),M(h.$$.fragment),T=d(),ie=o("span"),qe=s("GPT-Sw3"),Ee=d(),S=o("h2"),C=o("a"),ce=o("span"),M(V.$$.fragment),De=d(),pe=o("span"),Xe=s("Overview"),Pe=d(),j=o("p"),He=s(`The GPT-Sw3 model was first proposed in
`),B=o("a"),Qe=s("Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish"),Oe=s(`
by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey \xD6hman,
Fredrik Carlsson, Magnus Sahlgren.`),Me=d(),K=o("p"),Ye=s("Since that first paper the authors have extended their work and trained new models on their new 1.2TB corpora named The Nordic Pile."),ze=d(),ee=o("p"),Ke=s(`GPT-Sw3 is a collection of large decoder-only pretrained transformer language models that were developed by AI Sweden
in collaboration with RISE and the WASP WARA for Media and Language. GPT-Sw3 has been trained on a dataset containing
320B tokens in Swedish, Norwegian, Danish, Icelandic, English, and programming code. The model was pretrained using a
causal language modeling (CLM) objective utilizing the NeMo Megatron GPT implementation.`),$e=d(),I=o("p"),et=s("This model was contributed by "),W=o("a"),tt=s("AI Sweden"),nt=s("."),Ge=d(),f=o("p"),ot=s("The implementation uses the "),Z=o("a"),rt=s("GPT2Model"),at=s(` coupled
with our `),de=o("code"),st=s("GPTSw3Tokenizer"),lt=s(". This means that "),me=o("code"),it=s("AutoTokenizer"),ct=s(" and "),he=o("code"),pt=s("AutoModelForCausalLM"),dt=s(` map to our tokenizer
implementation and the corresponding GPT2 model implementation respectively.
`),fe=o("em"),mt=s("Note that sentencepiece is required to use our tokenizer and can be installed with:"),ht=d(),ue=o("code"),ft=s("pip install transformers[sentencepiece]"),ut=s(" or "),ge=o("code"),gt=s("pip install sentencepiece"),xe=d(),te=o("p"),wt=s("Example usage:"),Ae=d(),M(L.$$.fragment),Ue=d(),E=o("h2"),J=o("a"),we=o("span"),M(F.$$.fragment),vt=d(),ve=o("span"),_t=s("Documentation resources"),Ce=d(),b=o("ul"),_e=o("li"),ne=o("a"),kt=s("Text classification task guide"),Tt=d(),ke=o("li"),oe=o("a"),bt=s("Token classification task guide"),yt=d(),Te=o("li"),re=o("a"),St=s("Causal language modeling task guide"),je=d(),P=o("h2"),N=o("a"),be=o("span"),M(q.$$.fragment),Et=d(),ye=o("span"),Pt=s("GPTSw3Tokenizer"),Ie=d(),w=o("div"),M(D.$$.fragment),Mt=d(),X=o("p"),zt=s("Construct an GPTSw3 tokenizer. Based on "),H=o("a"),$t=s("SentencePiece"),Gt=s("."),xt=d(),Q=o("p"),At=s("This tokenizer inherits from "),ae=o("a"),Ut=s("PreTrainedTokenizer"),Ct=s(` which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`),jt=d(),M(R.$$.fragment),It=d(),se=o("div"),M(O.$$.fragment),this.h()},l(e){const i=bn('[data-svelte="svelte-1phssyn"]',document.head);u=r(i,"META",{name:!0,content:!0}),i.forEach(n),U=m(e),_=r(e,"H1",{class:!0});var Y=a(_);g=r(Y,"A",{id:!0,class:!0,href:!0});var Jt=a(g);k=r(Jt,"SPAN",{});var Nt=a(k);z(h.$$.fragment,Nt),Nt.forEach(n),Jt.forEach(n),T=m(Y),ie=r(Y,"SPAN",{});var Rt=a(ie);qe=l(Rt,"GPT-Sw3"),Rt.forEach(n),Y.forEach(n),Ee=m(e),S=r(e,"H2",{class:!0});var Ne=a(S);C=r(Ne,"A",{id:!0,class:!0,href:!0});var Vt=a(C);ce=r(Vt,"SPAN",{});var Bt=a(ce);z(V.$$.fragment,Bt),Bt.forEach(n),Vt.forEach(n),De=m(Ne),pe=r(Ne,"SPAN",{});var Wt=a(pe);Xe=l(Wt,"Overview"),Wt.forEach(n),Ne.forEach(n),Pe=m(e),j=r(e,"P",{});var Re=a(j);He=l(Re,`The GPT-Sw3 model was first proposed in
`),B=r(Re,"A",{href:!0,rel:!0});var Zt=a(B);Qe=l(Zt,"Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish"),Zt.forEach(n),Oe=l(Re,`
by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey \xD6hman,
Fredrik Carlsson, Magnus Sahlgren.`),Re.forEach(n),Me=m(e),K=r(e,"P",{});var Lt=a(K);Ye=l(Lt,"Since that first paper the authors have extended their work and trained new models on their new 1.2TB corpora named The Nordic Pile."),Lt.forEach(n),ze=m(e),ee=r(e,"P",{});var Ft=a(ee);Ke=l(Ft,`GPT-Sw3 is a collection of large decoder-only pretrained transformer language models that were developed by AI Sweden
in collaboration with RISE and the WASP WARA for Media and Language. GPT-Sw3 has been trained on a dataset containing
320B tokens in Swedish, Norwegian, Danish, Icelandic, English, and programming code. The model was pretrained using a
causal language modeling (CLM) objective utilizing the NeMo Megatron GPT implementation.`),Ft.forEach(n),$e=m(e),I=r(e,"P",{});var Ve=a(I);et=l(Ve,"This model was contributed by "),W=r(Ve,"A",{href:!0,rel:!0});var qt=a(W);tt=l(qt,"AI Sweden"),qt.forEach(n),nt=l(Ve,"."),Ve.forEach(n),Ge=m(e),f=r(e,"P",{});var v=a(f);ot=l(v,"The implementation uses the "),Z=r(v,"A",{href:!0,rel:!0});var Dt=a(Z);rt=l(Dt,"GPT2Model"),Dt.forEach(n),at=l(v,` coupled
with our `),de=r(v,"CODE",{});var Xt=a(de);st=l(Xt,"GPTSw3Tokenizer"),Xt.forEach(n),lt=l(v,". This means that "),me=r(v,"CODE",{});var Ht=a(me);it=l(Ht,"AutoTokenizer"),Ht.forEach(n),ct=l(v," and "),he=r(v,"CODE",{});var Qt=a(he);pt=l(Qt,"AutoModelForCausalLM"),Qt.forEach(n),dt=l(v,` map to our tokenizer
implementation and the corresponding GPT2 model implementation respectively.
`),fe=r(v,"EM",{});var Ot=a(fe);mt=l(Ot,"Note that sentencepiece is required to use our tokenizer and can be installed with:"),Ot.forEach(n),ht=m(v),ue=r(v,"CODE",{});var Yt=a(ue);ft=l(Yt,"pip install transformers[sentencepiece]"),Yt.forEach(n),ut=l(v," or "),ge=r(v,"CODE",{});var Kt=a(ge);gt=l(Kt,"pip install sentencepiece"),Kt.forEach(n),v.forEach(n),xe=m(e),te=r(e,"P",{});var en=a(te);wt=l(en,"Example usage:"),en.forEach(n),Ae=m(e),z(L.$$.fragment,e),Ue=m(e),E=r(e,"H2",{class:!0});var Be=a(E);J=r(Be,"A",{id:!0,class:!0,href:!0});var tn=a(J);we=r(tn,"SPAN",{});var nn=a(we);z(F.$$.fragment,nn),nn.forEach(n),tn.forEach(n),vt=m(Be),ve=r(Be,"SPAN",{});var on=a(ve);_t=l(on,"Documentation resources"),on.forEach(n),Be.forEach(n),Ce=m(e),b=r(e,"UL",{});var le=a(b);_e=r(le,"LI",{});var rn=a(_e);ne=r(rn,"A",{href:!0});var an=a(ne);kt=l(an,"Text classification task guide"),an.forEach(n),rn.forEach(n),Tt=m(le),ke=r(le,"LI",{});var sn=a(ke);oe=r(sn,"A",{href:!0});var ln=a(oe);bt=l(ln,"Token classification task guide"),ln.forEach(n),sn.forEach(n),yt=m(le),Te=r(le,"LI",{});var cn=a(Te);re=r(cn,"A",{href:!0});var pn=a(re);St=l(pn,"Causal language modeling task guide"),pn.forEach(n),cn.forEach(n),le.forEach(n),je=m(e),P=r(e,"H2",{class:!0});var We=a(P);N=r(We,"A",{id:!0,class:!0,href:!0});var dn=a(N);be=r(dn,"SPAN",{});var mn=a(be);z(q.$$.fragment,mn),mn.forEach(n),dn.forEach(n),Et=m(We),ye=r(We,"SPAN",{});var hn=a(ye);Pt=l(hn,"GPTSw3Tokenizer"),hn.forEach(n),We.forEach(n),Ie=m(e),w=r(e,"DIV",{class:!0});var y=a(w);z(D.$$.fragment,y),Mt=m(y),X=r(y,"P",{});var Ze=a(X);zt=l(Ze,"Construct an GPTSw3 tokenizer. Based on "),H=r(Ze,"A",{href:!0,rel:!0});var fn=a(H);$t=l(fn,"SentencePiece"),fn.forEach(n),Gt=l(Ze,"."),Ze.forEach(n),xt=m(y),Q=r(y,"P",{});var Le=a(Q);At=l(Le,"This tokenizer inherits from "),ae=r(Le,"A",{href:!0});var un=a(ae);Ut=l(un,"PreTrainedTokenizer"),un.forEach(n),Ct=l(Le,` which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`),Le.forEach(n),jt=m(y),z(R.$$.fragment,y),It=m(y),se=r(y,"DIV",{class:!0});var gn=a(se);z(O.$$.fragment,gn),gn.forEach(n),y.forEach(n),this.h()},h(){c(u,"name","hf:doc:metadata"),c(u,"content",JSON.stringify(zn)),c(g,"id","gptsw3"),c(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(g,"href","#gptsw3"),c(_,"class","relative group"),c(C,"id","overview"),c(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(C,"href","#overview"),c(S,"class","relative group"),c(B,"href","http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf"),c(B,"rel","nofollow"),c(W,"href","https://huggingface.co/AI-Sweden"),c(W,"rel","nofollow"),c(Z,"href","https://huggingface.co/docs/transformers/model_doc/gpt2"),c(Z,"rel","nofollow"),c(J,"id","documentation-resources"),c(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(J,"href","#documentation-resources"),c(E,"class","relative group"),c(ne,"href","../tasks/sequence_classification"),c(oe,"href","../tasks/token_classification"),c(re,"href","../tasks/language_modeling"),c(N,"id","transformers.GPTSw3Tokenizer"),c(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(N,"href","#transformers.GPTSw3Tokenizer"),c(P,"class","relative group"),c(H,"href","https://github.com/google/sentencepiece"),c(H,"rel","nofollow"),c(ae,"href","/docs/transformers/v4.31.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizer"),c(se,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,i){t(document.head,u),p(e,U,i),p(e,_,i),t(_,g),t(g,k),$(h,k,null),t(_,T),t(_,ie),t(ie,qe),p(e,Ee,i),p(e,S,i),t(S,C),t(C,ce),$(V,ce,null),t(S,De),t(S,pe),t(pe,Xe),p(e,Pe,i),p(e,j,i),t(j,He),t(j,B),t(B,Qe),t(j,Oe),p(e,Me,i),p(e,K,i),t(K,Ye),p(e,ze,i),p(e,ee,i),t(ee,Ke),p(e,$e,i),p(e,I,i),t(I,et),t(I,W),t(W,tt),t(I,nt),p(e,Ge,i),p(e,f,i),t(f,ot),t(f,Z),t(Z,rt),t(f,at),t(f,de),t(de,st),t(f,lt),t(f,me),t(me,it),t(f,ct),t(f,he),t(he,pt),t(f,dt),t(f,fe),t(fe,mt),t(f,ht),t(f,ue),t(ue,ft),t(f,ut),t(f,ge),t(ge,gt),p(e,xe,i),p(e,te,i),t(te,wt),p(e,Ae,i),$(L,e,i),p(e,Ue,i),p(e,E,i),t(E,J),t(J,we),$(F,we,null),t(E,vt),t(E,ve),t(ve,_t),p(e,Ce,i),p(e,b,i),t(b,_e),t(_e,ne),t(ne,kt),t(b,Tt),t(b,ke),t(ke,oe),t(oe,bt),t(b,yt),t(b,Te),t(Te,re),t(re,St),p(e,je,i),p(e,P,i),t(P,N),t(N,be),$(q,be,null),t(P,Et),t(P,ye),t(ye,Pt),p(e,Ie,i),p(e,w,i),$(D,w,null),t(w,Mt),t(w,X),t(X,zt),t(X,H),t(H,$t),t(X,Gt),t(w,xt),t(w,Q),t(Q,At),t(Q,ae),t(ae,Ut),t(Q,Ct),t(w,jt),$(R,w,null),t(w,It),t(w,se),$(O,se,null),Je=!0},p(e,[i]){const Y={};i&2&&(Y.$$scope={dirty:i,ctx:e}),R.$set(Y)},i(e){Je||(G(h.$$.fragment,e),G(V.$$.fragment,e),G(L.$$.fragment,e),G(F.$$.fragment,e),G(q.$$.fragment,e),G(D.$$.fragment,e),G(R.$$.fragment,e),G(O.$$.fragment,e),Je=!0)},o(e){x(h.$$.fragment,e),x(V.$$.fragment,e),x(L.$$.fragment,e),x(F.$$.fragment,e),x(q.$$.fragment,e),x(D.$$.fragment,e),x(R.$$.fragment,e),x(O.$$.fragment,e),Je=!1},d(e){n(u),e&&n(U),e&&n(_),A(h),e&&n(Ee),e&&n(S),A(V),e&&n(Pe),e&&n(j),e&&n(Me),e&&n(K),e&&n(ze),e&&n(ee),e&&n($e),e&&n(I),e&&n(Ge),e&&n(f),e&&n(xe),e&&n(te),e&&n(Ae),A(L,e),e&&n(Ue),e&&n(E),A(F),e&&n(Ce),e&&n(b),e&&n(je),e&&n(P),A(q),e&&n(Ie),e&&n(w),A(D),A(R),A(O)}}}const zn={local:"gptsw3",sections:[{local:"overview",title:"Overview"},{local:"documentation-resources",title:"Documentation resources"},{local:"transformers.GPTSw3Tokenizer",title:"GPTSw3Tokenizer"}],title:"GPT-Sw3"};function $n(Se){return yn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jn extends _n{constructor(u){super();kn(this,u,$n,Mn,Tn,{})}}export{jn as default,zn as metadata};
