import{S as Zt,i as Nt,s as Vt,e as r,k as m,w as V,t as o,M as Bt,c as a,d as t,m as f,a as n,x as B,h as i,b as c,G as s,g as p,y as F,L as Ft,q as S,o as P,B as I,v as St}from"../chunks/vendor-hf-doc-builder.js";import{I as nt}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as ge}from"../chunks/CodeBlock-hf-doc-builder.js";function Pt(it){let u,ie,M,_,H,E,we,Y,Ue,pe,j,x,Je,Ze,b,Ne,Ve,me,A,Be,fe,g,ce,Q,Fe,he,v,T,D,w,Se,K,Pe,ke,h,Ie,L,Ae,Qe,ee,We,Ce,de,U,ye,k,Ge,te,Xe,qe,W,Re,Oe,ue,z,$,se,J,He,le,Ye,Me,C,xe,je,Z,ve,d,De,re,Ke,Le,ae,et,tt,ze,N,_e,y,st,oe,lt,rt,G,at,ot,Te;return E=new nt({}),g=new ge({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`}}),w=new nt({}),U=new ge({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`}}),J=new nt({}),Z=new ge({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),N=new ge({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`}}),{c(){u=r("meta"),ie=m(),M=r("h1"),_=r("a"),H=r("span"),V(E.$$.fragment),we=m(),Y=r("span"),Ue=o("\u{1F917} Tokenizers \uB77C\uC774\uBE0C\uB7EC\uB9AC\uC758 \uD1A0\uD06C\uB098\uC774\uC800 \uC0AC\uC6A9\uD558\uAE30"),pe=m(),j=r("p"),x=r("code"),Je=o("PreTrainedTokenizerFast"),Ze=o("\uB294 "),b=r("a"),Ne=o("\u{1F917} Tokenizers"),Ve=o(` \uB77C\uC774\uBE0C\uB7EC\uB9AC\uC5D0 \uAE30\uBC18\uD569\uB2C8\uB2E4. \u{1F917} Tokenizers \uB77C\uC774\uBE0C\uB7EC\uB9AC\uC758 \uD1A0\uD06C\uB098\uC774\uC800\uB294
\u{1F917} Transformers\uB85C \uB9E4\uC6B0 \uAC04\uB2E8\uD558\uAC8C \uBD88\uB7EC\uC62C \uC218 \uC788\uC2B5\uB2C8\uB2E4.`),me=m(),A=r("p"),Be=o("\uAD6C\uCCB4\uC801\uC778 \uB0B4\uC6A9\uC5D0 \uB4E4\uC5B4\uAC00\uAE30 \uC804\uC5D0, \uBA87 \uC904\uC758 \uCF54\uB4DC\uB85C \uB354\uBBF8 \uD1A0\uD06C\uB098\uC774\uC800\uB97C \uB9CC\uB4E4\uC5B4 \uBCF4\uACA0\uC2B5\uB2C8\uB2E4:"),fe=m(),V(g.$$.fragment),ce=m(),Q=r("p"),Fe=o("\uC6B0\uB9AC\uAC00 \uC815\uC758\uD55C \uD30C\uC77C\uC744 \uD1B5\uD574 \uC774\uC81C \uD559\uC2B5\uB41C \uD1A0\uD06C\uB098\uC774\uC800\uB97C \uAC16\uAC8C \uB418\uC5C8\uC2B5\uB2C8\uB2E4. \uC774 \uB7F0\uD0C0\uC784\uC5D0\uC11C \uACC4\uC18D \uC0AC\uC6A9\uD558\uAC70\uB098 JSON \uD30C\uC77C\uB85C \uC800\uC7A5\uD558\uC5EC \uB098\uC911\uC5D0 \uC0AC\uC6A9\uD560 \uC218 \uC788\uC2B5\uB2C8\uB2E4."),he=m(),v=r("h2"),T=r("a"),D=r("span"),V(w.$$.fragment),Se=m(),K=r("span"),Pe=o("\uD1A0\uD06C\uB098\uC774\uC800 \uAC1D\uCCB4\uB85C\uBD80\uD130 \uC9C1\uC811 \uBD88\uB7EC\uC624\uAE30"),ke=m(),h=r("p"),Ie=o(`\u{1F917} Transformers \uB77C\uC774\uBE0C\uB7EC\uB9AC\uC5D0\uC11C \uC774 \uD1A0\uD06C\uB098\uC774\uC800 \uAC1D\uCCB4\uB97C \uD65C\uC6A9\uD558\uB294 \uBC29\uBC95\uC744 \uC0B4\uD3B4\uBCF4\uACA0\uC2B5\uB2C8\uB2E4.
`),L=r("code"),Ae=o("PreTrainedTokenizerFast"),Qe=o(" \uD074\uB798\uC2A4\uB294 \uC778\uC2A4\uD134\uC2A4\uD654\uB41C "),ee=r("em"),We=o("\uD1A0\uD06C\uB098\uC774\uC800"),Ce=o(" \uAC1D\uCCB4\uB97C \uC778\uC218\uB85C \uBC1B\uC544 \uC27D\uAC8C \uC778\uC2A4\uD134\uC2A4\uD654\uD560 \uC218 \uC788\uC2B5\uB2C8\uB2E4:"),de=m(),V(U.$$.fragment),ye=m(),k=r("p"),Ge=o("\uC774\uC81C "),te=r("code"),Xe=o("fast_tokenizer"),qe=o(" \uAC1D\uCCB4\uB294 \u{1F917} Transformers \uD1A0\uD06C\uB098\uC774\uC800\uC5D0\uC11C \uACF5\uC720\uD558\uB294 \uBAA8\uB4E0 \uBA54\uC18C\uB4DC\uC640 \uD568\uAED8 \uC0AC\uC6A9\uD560 \uC218 \uC788\uC2B5\uB2C8\uB2E4! \uC790\uC138\uD55C \uB0B4\uC6A9\uC740 "),W=r("a"),Re=o("\uD1A0\uD06C\uB098\uC774\uC800 \uD398\uC774\uC9C0"),Oe=o("\uB97C \uCC38\uC870\uD558\uC138\uC694."),ue=m(),z=r("h2"),$=r("a"),se=r("span"),V(J.$$.fragment),He=m(),le=r("span"),Ye=o("JSON \uD30C\uC77C\uC5D0\uC11C \uBD88\uB7EC\uC624\uAE30"),Me=m(),C=r("p"),xe=o("JSON \uD30C\uC77C\uC5D0\uC11C \uD1A0\uD06C\uB098\uC774\uC800\uB97C \uBD88\uB7EC\uC624\uAE30 \uC704\uD574, \uBA3C\uC800 \uD1A0\uD06C\uB098\uC774\uC800\uB97C \uC800\uC7A5\uD574 \uBCF4\uACA0\uC2B5\uB2C8\uB2E4:"),je=m(),V(Z.$$.fragment),ve=m(),d=r("p"),De=o("JSON \uD30C\uC77C\uC744 \uC800\uC7A5\uD55C \uACBD\uB85C\uB294 "),re=r("code"),Ke=o("tokenizer_file"),Le=o(" \uB9E4\uAC1C\uBCC0\uC218\uB97C \uC0AC\uC6A9\uD558\uC5EC "),ae=r("code"),et=o("PreTrainedTokenizerFast"),tt=o(" \uCD08\uAE30\uD654 \uBA54\uC18C\uB4DC\uC5D0 \uC804\uB2EC\uD560 \uC218 \uC788\uC2B5\uB2C8\uB2E4:"),ze=m(),V(N.$$.fragment),_e=m(),y=r("p"),st=o("\uC774\uC81C "),oe=r("code"),lt=o("fast_tokenizer"),rt=o(" \uAC1D\uCCB4\uB294 \u{1F917} Transformers \uD1A0\uD06C\uB098\uC774\uC800\uC5D0\uC11C \uACF5\uC720\uD558\uB294 \uBAA8\uB4E0 \uBA54\uC18C\uB4DC\uC640 \uD568\uAED8 \uC0AC\uC6A9\uD560 \uC218 \uC788\uC2B5\uB2C8\uB2E4! \uC790\uC138\uD55C \uB0B4\uC6A9\uC740 "),G=r("a"),at=o("\uD1A0\uD06C\uB098\uC774\uC800 \uD398\uC774\uC9C0"),ot=o("\uB97C \uCC38\uC870\uD558\uC138\uC694."),this.h()},l(e){const l=Bt('[data-svelte="svelte-1phssyn"]',document.head);u=a(l,"META",{name:!0,content:!0}),l.forEach(t),ie=f(e),M=a(e,"H1",{class:!0});var $e=n(M);_=a($e,"A",{id:!0,class:!0,href:!0});var pt=n(_);H=a(pt,"SPAN",{});var mt=n(H);B(E.$$.fragment,mt),mt.forEach(t),pt.forEach(t),we=f($e),Y=a($e,"SPAN",{});var ft=n(Y);Ue=i(ft,"\u{1F917} Tokenizers \uB77C\uC774\uBE0C\uB7EC\uB9AC\uC758 \uD1A0\uD06C\uB098\uC774\uC800 \uC0AC\uC6A9\uD558\uAE30"),ft.forEach(t),$e.forEach(t),pe=f(e),j=a(e,"P",{});var ne=n(j);x=a(ne,"CODE",{});var ct=n(x);Je=i(ct,"PreTrainedTokenizerFast"),ct.forEach(t),Ze=i(ne,"\uB294 "),b=a(ne,"A",{href:!0,rel:!0});var ht=n(b);Ne=i(ht,"\u{1F917} Tokenizers"),ht.forEach(t),Ve=i(ne,` \uB77C\uC774\uBE0C\uB7EC\uB9AC\uC5D0 \uAE30\uBC18\uD569\uB2C8\uB2E4. \u{1F917} Tokenizers \uB77C\uC774\uBE0C\uB7EC\uB9AC\uC758 \uD1A0\uD06C\uB098\uC774\uC800\uB294
\u{1F917} Transformers\uB85C \uB9E4\uC6B0 \uAC04\uB2E8\uD558\uAC8C \uBD88\uB7EC\uC62C \uC218 \uC788\uC2B5\uB2C8\uB2E4.`),ne.forEach(t),me=f(e),A=a(e,"P",{});var kt=n(A);Be=i(kt,"\uAD6C\uCCB4\uC801\uC778 \uB0B4\uC6A9\uC5D0 \uB4E4\uC5B4\uAC00\uAE30 \uC804\uC5D0, \uBA87 \uC904\uC758 \uCF54\uB4DC\uB85C \uB354\uBBF8 \uD1A0\uD06C\uB098\uC774\uC800\uB97C \uB9CC\uB4E4\uC5B4 \uBCF4\uACA0\uC2B5\uB2C8\uB2E4:"),kt.forEach(t),fe=f(e),B(g.$$.fragment,e),ce=f(e),Q=a(e,"P",{});var dt=n(Q);Fe=i(dt,"\uC6B0\uB9AC\uAC00 \uC815\uC758\uD55C \uD30C\uC77C\uC744 \uD1B5\uD574 \uC774\uC81C \uD559\uC2B5\uB41C \uD1A0\uD06C\uB098\uC774\uC800\uB97C \uAC16\uAC8C \uB418\uC5C8\uC2B5\uB2C8\uB2E4. \uC774 \uB7F0\uD0C0\uC784\uC5D0\uC11C \uACC4\uC18D \uC0AC\uC6A9\uD558\uAC70\uB098 JSON \uD30C\uC77C\uB85C \uC800\uC7A5\uD558\uC5EC \uB098\uC911\uC5D0 \uC0AC\uC6A9\uD560 \uC218 \uC788\uC2B5\uB2C8\uB2E4."),dt.forEach(t),he=f(e),v=a(e,"H2",{class:!0});var Ee=n(v);T=a(Ee,"A",{id:!0,class:!0,href:!0});var yt=n(T);D=a(yt,"SPAN",{});var ut=n(D);B(w.$$.fragment,ut),ut.forEach(t),yt.forEach(t),Se=f(Ee),K=a(Ee,"SPAN",{});var Mt=n(K);Pe=i(Mt,"\uD1A0\uD06C\uB098\uC774\uC800 \uAC1D\uCCB4\uB85C\uBD80\uD130 \uC9C1\uC811 \uBD88\uB7EC\uC624\uAE30"),Mt.forEach(t),Ee.forEach(t),ke=f(e),h=a(e,"P",{});var X=n(h);Ie=i(X,`\u{1F917} Transformers \uB77C\uC774\uBE0C\uB7EC\uB9AC\uC5D0\uC11C \uC774 \uD1A0\uD06C\uB098\uC774\uC800 \uAC1D\uCCB4\uB97C \uD65C\uC6A9\uD558\uB294 \uBC29\uBC95\uC744 \uC0B4\uD3B4\uBCF4\uACA0\uC2B5\uB2C8\uB2E4.
`),L=a(X,"CODE",{});var jt=n(L);Ae=i(jt,"PreTrainedTokenizerFast"),jt.forEach(t),Qe=i(X," \uD074\uB798\uC2A4\uB294 \uC778\uC2A4\uD134\uC2A4\uD654\uB41C "),ee=a(X,"EM",{});var vt=n(ee);We=i(vt,"\uD1A0\uD06C\uB098\uC774\uC800"),vt.forEach(t),Ce=i(X," \uAC1D\uCCB4\uB97C \uC778\uC218\uB85C \uBC1B\uC544 \uC27D\uAC8C \uC778\uC2A4\uD134\uC2A4\uD654\uD560 \uC218 \uC788\uC2B5\uB2C8\uB2E4:"),X.forEach(t),de=f(e),B(U.$$.fragment,e),ye=f(e),k=a(e,"P",{});var q=n(k);Ge=i(q,"\uC774\uC81C "),te=a(q,"CODE",{});var zt=n(te);Xe=i(zt,"fast_tokenizer"),zt.forEach(t),qe=i(q," \uAC1D\uCCB4\uB294 \u{1F917} Transformers \uD1A0\uD06C\uB098\uC774\uC800\uC5D0\uC11C \uACF5\uC720\uD558\uB294 \uBAA8\uB4E0 \uBA54\uC18C\uB4DC\uC640 \uD568\uAED8 \uC0AC\uC6A9\uD560 \uC218 \uC788\uC2B5\uB2C8\uB2E4! \uC790\uC138\uD55C \uB0B4\uC6A9\uC740 "),W=a(q,"A",{href:!0});var _t=n(W);Re=i(_t,"\uD1A0\uD06C\uB098\uC774\uC800 \uD398\uC774\uC9C0"),_t.forEach(t),Oe=i(q,"\uB97C \uCC38\uC870\uD558\uC138\uC694."),q.forEach(t),ue=f(e),z=a(e,"H2",{class:!0});var be=n(z);$=a(be,"A",{id:!0,class:!0,href:!0});var Tt=n($);se=a(Tt,"SPAN",{});var $t=n(se);B(J.$$.fragment,$t),$t.forEach(t),Tt.forEach(t),He=f(be),le=a(be,"SPAN",{});var Et=n(le);Ye=i(Et,"JSON \uD30C\uC77C\uC5D0\uC11C \uBD88\uB7EC\uC624\uAE30"),Et.forEach(t),be.forEach(t),Me=f(e),C=a(e,"P",{});var bt=n(C);xe=i(bt,"JSON \uD30C\uC77C\uC5D0\uC11C \uD1A0\uD06C\uB098\uC774\uC800\uB97C \uBD88\uB7EC\uC624\uAE30 \uC704\uD574, \uBA3C\uC800 \uD1A0\uD06C\uB098\uC774\uC800\uB97C \uC800\uC7A5\uD574 \uBCF4\uACA0\uC2B5\uB2C8\uB2E4:"),bt.forEach(t),je=f(e),B(Z.$$.fragment,e),ve=f(e),d=a(e,"P",{});var R=n(d);De=i(R,"JSON \uD30C\uC77C\uC744 \uC800\uC7A5\uD55C \uACBD\uB85C\uB294 "),re=a(R,"CODE",{});var gt=n(re);Ke=i(gt,"tokenizer_file"),gt.forEach(t),Le=i(R," \uB9E4\uAC1C\uBCC0\uC218\uB97C \uC0AC\uC6A9\uD558\uC5EC "),ae=a(R,"CODE",{});var wt=n(ae);et=i(wt,"PreTrainedTokenizerFast"),wt.forEach(t),tt=i(R," \uCD08\uAE30\uD654 \uBA54\uC18C\uB4DC\uC5D0 \uC804\uB2EC\uD560 \uC218 \uC788\uC2B5\uB2C8\uB2E4:"),R.forEach(t),ze=f(e),B(N.$$.fragment,e),_e=f(e),y=a(e,"P",{});var O=n(y);st=i(O,"\uC774\uC81C "),oe=a(O,"CODE",{});var Ut=n(oe);lt=i(Ut,"fast_tokenizer"),Ut.forEach(t),rt=i(O," \uAC1D\uCCB4\uB294 \u{1F917} Transformers \uD1A0\uD06C\uB098\uC774\uC800\uC5D0\uC11C \uACF5\uC720\uD558\uB294 \uBAA8\uB4E0 \uBA54\uC18C\uB4DC\uC640 \uD568\uAED8 \uC0AC\uC6A9\uD560 \uC218 \uC788\uC2B5\uB2C8\uB2E4! \uC790\uC138\uD55C \uB0B4\uC6A9\uC740 "),G=a(O,"A",{href:!0});var Jt=n(G);at=i(Jt,"\uD1A0\uD06C\uB098\uC774\uC800 \uD398\uC774\uC9C0"),Jt.forEach(t),ot=i(O,"\uB97C \uCC38\uC870\uD558\uC138\uC694."),O.forEach(t),this.h()},h(){c(u,"name","hf:doc:metadata"),c(u,"content",JSON.stringify(It)),c(_,"id","use-tokenizers-from-tokenizers"),c(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(_,"href","#use-tokenizers-from-tokenizers"),c(M,"class","relative group"),c(b,"href","https://huggingface.co/docs/tokenizers"),c(b,"rel","nofollow"),c(T,"id","loading-directly-from-the-tokenizer-object"),c(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(T,"href","#loading-directly-from-the-tokenizer-object"),c(v,"class","relative group"),c(W,"href","main_classes/tokenizer"),c($,"id","loading-from-a-JSON-file"),c($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c($,"href","#loading-from-a-JSON-file"),c(z,"class","relative group"),c(G,"href","main_classes/tokenizer")},m(e,l){s(document.head,u),p(e,ie,l),p(e,M,l),s(M,_),s(_,H),F(E,H,null),s(M,we),s(M,Y),s(Y,Ue),p(e,pe,l),p(e,j,l),s(j,x),s(x,Je),s(j,Ze),s(j,b),s(b,Ne),s(j,Ve),p(e,me,l),p(e,A,l),s(A,Be),p(e,fe,l),F(g,e,l),p(e,ce,l),p(e,Q,l),s(Q,Fe),p(e,he,l),p(e,v,l),s(v,T),s(T,D),F(w,D,null),s(v,Se),s(v,K),s(K,Pe),p(e,ke,l),p(e,h,l),s(h,Ie),s(h,L),s(L,Ae),s(h,Qe),s(h,ee),s(ee,We),s(h,Ce),p(e,de,l),F(U,e,l),p(e,ye,l),p(e,k,l),s(k,Ge),s(k,te),s(te,Xe),s(k,qe),s(k,W),s(W,Re),s(k,Oe),p(e,ue,l),p(e,z,l),s(z,$),s($,se),F(J,se,null),s(z,He),s(z,le),s(le,Ye),p(e,Me,l),p(e,C,l),s(C,xe),p(e,je,l),F(Z,e,l),p(e,ve,l),p(e,d,l),s(d,De),s(d,re),s(re,Ke),s(d,Le),s(d,ae),s(ae,et),s(d,tt),p(e,ze,l),F(N,e,l),p(e,_e,l),p(e,y,l),s(y,st),s(y,oe),s(oe,lt),s(y,rt),s(y,G),s(G,at),s(y,ot),Te=!0},p:Ft,i(e){Te||(S(E.$$.fragment,e),S(g.$$.fragment,e),S(w.$$.fragment,e),S(U.$$.fragment,e),S(J.$$.fragment,e),S(Z.$$.fragment,e),S(N.$$.fragment,e),Te=!0)},o(e){P(E.$$.fragment,e),P(g.$$.fragment,e),P(w.$$.fragment,e),P(U.$$.fragment,e),P(J.$$.fragment,e),P(Z.$$.fragment,e),P(N.$$.fragment,e),Te=!1},d(e){t(u),e&&t(ie),e&&t(M),I(E),e&&t(pe),e&&t(j),e&&t(me),e&&t(A),e&&t(fe),I(g,e),e&&t(ce),e&&t(Q),e&&t(he),e&&t(v),I(w),e&&t(ke),e&&t(h),e&&t(de),I(U,e),e&&t(ye),e&&t(k),e&&t(ue),e&&t(z),I(J),e&&t(Me),e&&t(C),e&&t(je),I(Z,e),e&&t(ve),e&&t(d),e&&t(ze),I(N,e),e&&t(_e),e&&t(y)}}}const It={local:"use-tokenizers-from-tokenizers",sections:[{local:"loading-directly-from-the-tokenizer-object",title:"\uD1A0\uD06C\uB098\uC774\uC800 \uAC1D\uCCB4\uB85C\uBD80\uD130 \uC9C1\uC811 \uBD88\uB7EC\uC624\uAE30"},{local:"loading-from-a-JSON-file",title:"JSON \uD30C\uC77C\uC5D0\uC11C \uBD88\uB7EC\uC624\uAE30"}],title:"\u{1F917} Tokenizers \uB77C\uC774\uBE0C\uB7EC\uB9AC\uC758 \uD1A0\uD06C\uB098\uC774\uC800 \uC0AC\uC6A9\uD558\uAE30"};function At(it){return St(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Gt extends Zt{constructor(u){super();Nt(this,u,At,Pt,Vt,{})}}export{Gt as default,It as metadata};
