import{S as Mt,i as wt,s as _t,e as a,k as m,w as F,t as l,M as $t,c as n,d as t,m as h,a as o,x as N,h as i,b as p,G as s,g as f,y as B,L as Ut,q as P,o as I,B as W,v as Et}from"../chunks/vendor-hf-doc-builder.js";import{I as st}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as we}from"../chunks/CodeBlock-hf-doc-builder.js";function Jt(rt){let y,ae,u,g,Y,w,_e,x,$e,ne,c,Ue,A,Ee,Je,_,Ze,Ve,oe,S,Fe,le,$,ie,Q,Ne,fe,z,j,K,U,Be,D,Pe,pe,d,Ie,G,We,Ae,ee,Se,Qe,me,E,he,v,Ge,X,Xe,qe,ce,b,T,te,J,Re,se,Ce,de,q,Le,ke,Z,ye,k,He,R,Oe,Ye,re,xe,Ke,ue,V,ze,M,De,C,et,tt,be;return w=new st({}),$=new we({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`}}),U=new st({}),E=new we({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`}}),J=new st({}),Z=new we({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),V=new we({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`}}),{c(){y=a("meta"),ae=m(),u=a("h1"),g=a("a"),Y=a("span"),F(w.$$.fragment),_e=m(),x=a("span"),$e=l("Use tokenizers from \u{1F917} Tokenizers"),ne=m(),c=a("p"),Ue=l("The "),A=a("a"),Ee=l("PreTrainedTokenizerFast"),Je=l(" depends on the "),_=a("a"),Ze=l("\u{1F917} Tokenizers"),Ve=l(` library. The tokenizers obtained from the \u{1F917} Tokenizers library can be
loaded very simply into \u{1F917} Transformers.`),oe=m(),S=a("p"),Fe=l("Before getting in the specifics, let\u2019s first start by creating a dummy tokenizer in a few lines:"),le=m(),F($.$$.fragment),ie=m(),Q=a("p"),Ne=l(`We now have a tokenizer trained on the files we defined. We can either continue using it in that runtime, or save it to
a JSON file for future re-use.`),fe=m(),z=a("h2"),j=a("a"),K=a("span"),F(U.$$.fragment),Be=m(),D=a("span"),Pe=l("Loading directly from the tokenizer object"),pe=m(),d=a("p"),Ie=l(`Let\u2019s see how to leverage this tokenizer object in the \u{1F917} Transformers library. The
`),G=a("a"),We=l("PreTrainedTokenizerFast"),Ae=l(` class allows for easy instantiation, by accepting the instantiated
`),ee=a("em"),Se=l("tokenizer"),Qe=l(" object as an argument:"),me=m(),F(E.$$.fragment),he=m(),v=a("p"),Ge=l("This object can now be used with all the methods shared by the \u{1F917} Transformers tokenizers! Head to "),X=a("a"),Xe=l(`the tokenizer
page`),qe=l(" for more information."),ce=m(),b=a("h2"),T=a("a"),te=a("span"),F(J.$$.fragment),Re=m(),se=a("span"),Ce=l("Loading from a JSON file"),de=m(),q=a("p"),Le=l("In order to load a tokenizer from a JSON file, let\u2019s first start by saving our tokenizer:"),ke=m(),F(Z.$$.fragment),ye=m(),k=a("p"),He=l("The path to which we saved this file can be passed to the "),R=a("a"),Oe=l("PreTrainedTokenizerFast"),Ye=l(` initialization
method using the `),re=a("code"),xe=l("tokenizer_file"),Ke=l(" parameter:"),ue=m(),F(V.$$.fragment),ze=m(),M=a("p"),De=l("This object can now be used with all the methods shared by the \u{1F917} Transformers tokenizers! Head to "),C=a("a"),et=l(`the tokenizer
page`),tt=l(" for more information."),this.h()},l(e){const r=$t('[data-svelte="svelte-1phssyn"]',document.head);y=n(r,"META",{name:!0,content:!0}),r.forEach(t),ae=h(e),u=n(e,"H1",{class:!0});var ge=o(u);g=n(ge,"A",{id:!0,class:!0,href:!0});var at=o(g);Y=n(at,"SPAN",{});var nt=o(Y);N(w.$$.fragment,nt),nt.forEach(t),at.forEach(t),_e=h(ge),x=n(ge,"SPAN",{});var ot=o(x);$e=i(ot,"Use tokenizers from \u{1F917} Tokenizers"),ot.forEach(t),ge.forEach(t),ne=h(e),c=n(e,"P",{});var L=o(c);Ue=i(L,"The "),A=n(L,"A",{href:!0});var lt=o(A);Ee=i(lt,"PreTrainedTokenizerFast"),lt.forEach(t),Je=i(L," depends on the "),_=n(L,"A",{href:!0,rel:!0});var it=o(_);Ze=i(it,"\u{1F917} Tokenizers"),it.forEach(t),Ve=i(L,` library. The tokenizers obtained from the \u{1F917} Tokenizers library can be
loaded very simply into \u{1F917} Transformers.`),L.forEach(t),oe=h(e),S=n(e,"P",{});var ft=o(S);Fe=i(ft,"Before getting in the specifics, let\u2019s first start by creating a dummy tokenizer in a few lines:"),ft.forEach(t),le=h(e),N($.$$.fragment,e),ie=h(e),Q=n(e,"P",{});var pt=o(Q);Ne=i(pt,`We now have a tokenizer trained on the files we defined. We can either continue using it in that runtime, or save it to
a JSON file for future re-use.`),pt.forEach(t),fe=h(e),z=n(e,"H2",{class:!0});var je=o(z);j=n(je,"A",{id:!0,class:!0,href:!0});var mt=o(j);K=n(mt,"SPAN",{});var ht=o(K);N(U.$$.fragment,ht),ht.forEach(t),mt.forEach(t),Be=h(je),D=n(je,"SPAN",{});var ct=o(D);Pe=i(ct,"Loading directly from the tokenizer object"),ct.forEach(t),je.forEach(t),pe=h(e),d=n(e,"P",{});var H=o(d);Ie=i(H,`Let\u2019s see how to leverage this tokenizer object in the \u{1F917} Transformers library. The
`),G=n(H,"A",{href:!0});var dt=o(G);We=i(dt,"PreTrainedTokenizerFast"),dt.forEach(t),Ae=i(H,` class allows for easy instantiation, by accepting the instantiated
`),ee=n(H,"EM",{});var kt=o(ee);Se=i(kt,"tokenizer"),kt.forEach(t),Qe=i(H," object as an argument:"),H.forEach(t),me=h(e),N(E.$$.fragment,e),he=h(e),v=n(e,"P",{});var ve=o(v);Ge=i(ve,"This object can now be used with all the methods shared by the \u{1F917} Transformers tokenizers! Head to "),X=n(ve,"A",{href:!0});var yt=o(X);Xe=i(yt,`the tokenizer
page`),yt.forEach(t),qe=i(ve," for more information."),ve.forEach(t),ce=h(e),b=n(e,"H2",{class:!0});var Te=o(b);T=n(Te,"A",{id:!0,class:!0,href:!0});var ut=o(T);te=n(ut,"SPAN",{});var zt=o(te);N(J.$$.fragment,zt),zt.forEach(t),ut.forEach(t),Re=h(Te),se=n(Te,"SPAN",{});var bt=o(se);Ce=i(bt,"Loading from a JSON file"),bt.forEach(t),Te.forEach(t),de=h(e),q=n(e,"P",{});var gt=o(q);Le=i(gt,"In order to load a tokenizer from a JSON file, let\u2019s first start by saving our tokenizer:"),gt.forEach(t),ke=h(e),N(Z.$$.fragment,e),ye=h(e),k=n(e,"P",{});var O=o(k);He=i(O,"The path to which we saved this file can be passed to the "),R=n(O,"A",{href:!0});var jt=o(R);Oe=i(jt,"PreTrainedTokenizerFast"),jt.forEach(t),Ye=i(O,` initialization
method using the `),re=n(O,"CODE",{});var vt=o(re);xe=i(vt,"tokenizer_file"),vt.forEach(t),Ke=i(O," parameter:"),O.forEach(t),ue=h(e),N(V.$$.fragment,e),ze=h(e),M=n(e,"P",{});var Me=o(M);De=i(Me,"This object can now be used with all the methods shared by the \u{1F917} Transformers tokenizers! Head to "),C=n(Me,"A",{href:!0});var Tt=o(C);et=i(Tt,`the tokenizer
page`),Tt.forEach(t),tt=i(Me," for more information."),Me.forEach(t),this.h()},h(){p(y,"name","hf:doc:metadata"),p(y,"content",JSON.stringify(Zt)),p(g,"id","use-tokenizers-from-tokenizers"),p(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(g,"href","#use-tokenizers-from-tokenizers"),p(u,"class","relative group"),p(A,"href","/docs/transformers/v4.31.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast"),p(_,"href","https://huggingface.co/docs/tokenizers"),p(_,"rel","nofollow"),p(j,"id","loading-directly-from-the-tokenizer-object"),p(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(j,"href","#loading-directly-from-the-tokenizer-object"),p(z,"class","relative group"),p(G,"href","/docs/transformers/v4.31.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast"),p(X,"href","main_classes/tokenizer"),p(T,"id","loading-from-a-json-file"),p(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(T,"href","#loading-from-a-json-file"),p(b,"class","relative group"),p(R,"href","/docs/transformers/v4.31.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast"),p(C,"href","main_classes/tokenizer")},m(e,r){s(document.head,y),f(e,ae,r),f(e,u,r),s(u,g),s(g,Y),B(w,Y,null),s(u,_e),s(u,x),s(x,$e),f(e,ne,r),f(e,c,r),s(c,Ue),s(c,A),s(A,Ee),s(c,Je),s(c,_),s(_,Ze),s(c,Ve),f(e,oe,r),f(e,S,r),s(S,Fe),f(e,le,r),B($,e,r),f(e,ie,r),f(e,Q,r),s(Q,Ne),f(e,fe,r),f(e,z,r),s(z,j),s(j,K),B(U,K,null),s(z,Be),s(z,D),s(D,Pe),f(e,pe,r),f(e,d,r),s(d,Ie),s(d,G),s(G,We),s(d,Ae),s(d,ee),s(ee,Se),s(d,Qe),f(e,me,r),B(E,e,r),f(e,he,r),f(e,v,r),s(v,Ge),s(v,X),s(X,Xe),s(v,qe),f(e,ce,r),f(e,b,r),s(b,T),s(T,te),B(J,te,null),s(b,Re),s(b,se),s(se,Ce),f(e,de,r),f(e,q,r),s(q,Le),f(e,ke,r),B(Z,e,r),f(e,ye,r),f(e,k,r),s(k,He),s(k,R),s(R,Oe),s(k,Ye),s(k,re),s(re,xe),s(k,Ke),f(e,ue,r),B(V,e,r),f(e,ze,r),f(e,M,r),s(M,De),s(M,C),s(C,et),s(M,tt),be=!0},p:Ut,i(e){be||(P(w.$$.fragment,e),P($.$$.fragment,e),P(U.$$.fragment,e),P(E.$$.fragment,e),P(J.$$.fragment,e),P(Z.$$.fragment,e),P(V.$$.fragment,e),be=!0)},o(e){I(w.$$.fragment,e),I($.$$.fragment,e),I(U.$$.fragment,e),I(E.$$.fragment,e),I(J.$$.fragment,e),I(Z.$$.fragment,e),I(V.$$.fragment,e),be=!1},d(e){t(y),e&&t(ae),e&&t(u),W(w),e&&t(ne),e&&t(c),e&&t(oe),e&&t(S),e&&t(le),W($,e),e&&t(ie),e&&t(Q),e&&t(fe),e&&t(z),W(U),e&&t(pe),e&&t(d),e&&t(me),W(E,e),e&&t(he),e&&t(v),e&&t(ce),e&&t(b),W(J),e&&t(de),e&&t(q),e&&t(ke),W(Z,e),e&&t(ye),e&&t(k),e&&t(ue),W(V,e),e&&t(ze),e&&t(M)}}}const Zt={local:"use-tokenizers-from-tokenizers",sections:[{local:"loading-directly-from-the-tokenizer-object",title:"Loading directly from the tokenizer object"},{local:"loading-from-a-json-file",title:"Loading from a JSON file"}],title:"Use tokenizers from \u{1F917} Tokenizers"};function Vt(rt){return Et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pt extends Mt{constructor(y){super();wt(this,y,Vt,Jt,_t,{})}}export{Pt as default,Zt as metadata};
