import{S as Ta,i as _a,s as $a,e as t,k as m,w as q,t as n,M as Ea,c as r,d as a,m as d,a as l,x as N,h as i,b as c,G as o,g as p,y as A,L as Ua,q as B,o as F,B as P,v as wa}from"../chunks/vendor-hf-doc-builder.js";import{I as oa}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as _e}from"../chunks/CodeBlock-hf-doc-builder.js";function Va(sa){let k,te,z,j,O,_,$e,x,Ee,re,f,Ue,H,we,Ve,$,Je,Ze,le,I,qe,ne,E,ie,S,Ne,pe,v,b,Y,U,Ae,K,Be,me,u,Fe,L,Pe,Ie,D,Se,Ce,de,w,ce,M,Qe,C,We,Ge,fe,y,g,ee,V,Xe,ae,Re,ue,Q,Oe,he,J,ke,h,xe,oe,He,Ye,se,Ke,Le,ze,Z,ve,T,De,W,ea,aa,ye;return _=new oa({}),E=new _e({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`}}),U=new oa({}),w=new _e({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`}}),V=new oa({}),J=new _e({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),Z=new _e({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`}}),{c(){k=t("meta"),te=m(),z=t("h1"),j=t("a"),O=t("span"),q(_.$$.fragment),$e=m(),x=t("span"),Ee=n("Usando os Tokenizers do \u{1F917} Tokenizers"),re=m(),f=t("p"),Ue=n("O "),H=t("code"),we=n("PreTrainedTokenizerFast"),Ve=n(" depende da biblioteca "),$=t("a"),Je=n("\u{1F917} Tokenizers"),Ze=n(". O Tokenizer obtido da biblioteca \u{1F917} Tokenizers pode ser carregado facilmente pelo \u{1F917} Transformers."),le=m(),I=t("p"),qe=n("Antes de entrar nos detalhes, vamos come\xE7ar criando um tokenizer fict\xEDcio em algumas linhas:"),ne=m(),q(E.$$.fragment),ie=m(),S=t("p"),Ne=n("Agora temos um tokenizer treinado nos arquivos que foram definidos. N\xF3s podemos continuar usando nessa execu\xE7\xE3o ou salvar em um arquivo JSON para re-utilizar no futuro."),pe=m(),v=t("h2"),b=t("a"),Y=t("span"),q(U.$$.fragment),Ae=m(),K=t("span"),Be=n("Carregando diretamente de um objeto tokenizer"),me=m(),u=t("p"),Fe=n("Vamos ver como aproveitar esse objeto tokenizer na biblioteca \u{1F917} Transformers. A classe "),L=t("code"),Pe=n("PreTrainedTokenizerFast"),Ie=n(" permite uma instancia\xE7\xE3o f\xE1cil, aceitando o objeto "),D=t("em"),Se=n("tokenizer"),Ce=n(" instanciado como um argumento:"),de=m(),q(w.$$.fragment),ce=m(),M=t("p"),Qe=n("Esse objeto pode ser utilizado com todos os m\xE9todos compartilhados pelos tokenizers dos \u{1F917} Transformers! V\xE1 para "),C=t("a"),We=n("a p\xE1gina do tokenizer"),Ge=n(" para mais informa\xE7\xF5es."),fe=m(),y=t("h2"),g=t("a"),ee=t("span"),q(V.$$.fragment),Xe=m(),ae=t("span"),Re=n("Carregando de um arquivo JSON"),ue=m(),Q=t("p"),Oe=n("Para carregar um tokenizer de um arquivo JSON vamos primeiro come\xE7ar salvando nosso tokenizer:"),he=m(),q(J.$$.fragment),ke=m(),h=t("p"),xe=n("A pasta para qual salvamos esse arquivo pode ser passada para o m\xE9todo de inicializa\xE7\xE3o do "),oe=t("code"),He=n("PreTrainedTokenizerFast"),Ye=n(" usando o "),se=t("code"),Ke=n("tokenizer_file"),Le=n(" par\xE2metro:"),ze=m(),q(Z.$$.fragment),ve=m(),T=t("p"),De=n("Esse objeto pode ser utilizado com todos os m\xE9todos compartilhados pelos tokenizers dos \u{1F917} Transformers! V\xE1 para "),W=t("a"),ea=n("a p\xE1gina do tokenizer"),aa=n(" para mais informa\xE7\xF5es."),this.h()},l(e){const s=Ea('[data-svelte="svelte-1phssyn"]',document.head);k=r(s,"META",{name:!0,content:!0}),s.forEach(a),te=d(e),z=r(e,"H1",{class:!0});var je=l(z);j=r(je,"A",{id:!0,class:!0,href:!0});var ta=l(j);O=r(ta,"SPAN",{});var ra=l(O);N(_.$$.fragment,ra),ra.forEach(a),ta.forEach(a),$e=d(je),x=r(je,"SPAN",{});var la=l(x);Ee=i(la,"Usando os Tokenizers do \u{1F917} Tokenizers"),la.forEach(a),je.forEach(a),re=d(e),f=r(e,"P",{});var G=l(f);Ue=i(G,"O "),H=r(G,"CODE",{});var na=l(H);we=i(na,"PreTrainedTokenizerFast"),na.forEach(a),Ve=i(G," depende da biblioteca "),$=r(G,"A",{href:!0,rel:!0});var ia=l($);Je=i(ia,"\u{1F917} Tokenizers"),ia.forEach(a),Ze=i(G,". O Tokenizer obtido da biblioteca \u{1F917} Tokenizers pode ser carregado facilmente pelo \u{1F917} Transformers."),G.forEach(a),le=d(e),I=r(e,"P",{});var pa=l(I);qe=i(pa,"Antes de entrar nos detalhes, vamos come\xE7ar criando um tokenizer fict\xEDcio em algumas linhas:"),pa.forEach(a),ne=d(e),N(E.$$.fragment,e),ie=d(e),S=r(e,"P",{});var ma=l(S);Ne=i(ma,"Agora temos um tokenizer treinado nos arquivos que foram definidos. N\xF3s podemos continuar usando nessa execu\xE7\xE3o ou salvar em um arquivo JSON para re-utilizar no futuro."),ma.forEach(a),pe=d(e),v=r(e,"H2",{class:!0});var be=l(v);b=r(be,"A",{id:!0,class:!0,href:!0});var da=l(b);Y=r(da,"SPAN",{});var ca=l(Y);N(U.$$.fragment,ca),ca.forEach(a),da.forEach(a),Ae=d(be),K=r(be,"SPAN",{});var fa=l(K);Be=i(fa,"Carregando diretamente de um objeto tokenizer"),fa.forEach(a),be.forEach(a),me=d(e),u=r(e,"P",{});var X=l(u);Fe=i(X,"Vamos ver como aproveitar esse objeto tokenizer na biblioteca \u{1F917} Transformers. A classe "),L=r(X,"CODE",{});var ua=l(L);Pe=i(ua,"PreTrainedTokenizerFast"),ua.forEach(a),Ie=i(X," permite uma instancia\xE7\xE3o f\xE1cil, aceitando o objeto "),D=r(X,"EM",{});var ha=l(D);Se=i(ha,"tokenizer"),ha.forEach(a),Ce=i(X," instanciado como um argumento:"),X.forEach(a),de=d(e),N(w.$$.fragment,e),ce=d(e),M=r(e,"P",{});var Me=l(M);Qe=i(Me,"Esse objeto pode ser utilizado com todos os m\xE9todos compartilhados pelos tokenizers dos \u{1F917} Transformers! V\xE1 para "),C=r(Me,"A",{href:!0});var ka=l(C);We=i(ka,"a p\xE1gina do tokenizer"),ka.forEach(a),Ge=i(Me," para mais informa\xE7\xF5es."),Me.forEach(a),fe=d(e),y=r(e,"H2",{class:!0});var ge=l(y);g=r(ge,"A",{id:!0,class:!0,href:!0});var za=l(g);ee=r(za,"SPAN",{});var va=l(ee);N(V.$$.fragment,va),va.forEach(a),za.forEach(a),Xe=d(ge),ae=r(ge,"SPAN",{});var ya=l(ae);Re=i(ya,"Carregando de um arquivo JSON"),ya.forEach(a),ge.forEach(a),ue=d(e),Q=r(e,"P",{});var ja=l(Q);Oe=i(ja,"Para carregar um tokenizer de um arquivo JSON vamos primeiro come\xE7ar salvando nosso tokenizer:"),ja.forEach(a),he=d(e),N(J.$$.fragment,e),ke=d(e),h=r(e,"P",{});var R=l(h);xe=i(R,"A pasta para qual salvamos esse arquivo pode ser passada para o m\xE9todo de inicializa\xE7\xE3o do "),oe=r(R,"CODE",{});var ba=l(oe);He=i(ba,"PreTrainedTokenizerFast"),ba.forEach(a),Ye=i(R," usando o "),se=r(R,"CODE",{});var Ma=l(se);Ke=i(Ma,"tokenizer_file"),Ma.forEach(a),Le=i(R," par\xE2metro:"),R.forEach(a),ze=d(e),N(Z.$$.fragment,e),ve=d(e),T=r(e,"P",{});var Te=l(T);De=i(Te,"Esse objeto pode ser utilizado com todos os m\xE9todos compartilhados pelos tokenizers dos \u{1F917} Transformers! V\xE1 para "),W=r(Te,"A",{href:!0});var ga=l(W);ea=i(ga,"a p\xE1gina do tokenizer"),ga.forEach(a),aa=i(Te," para mais informa\xE7\xF5es."),Te.forEach(a),this.h()},h(){c(k,"name","hf:doc:metadata"),c(k,"content",JSON.stringify(Ja)),c(j,"id","usando-os-tokenizers-do-tokenizers"),c(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(j,"href","#usando-os-tokenizers-do-tokenizers"),c(z,"class","relative group"),c($,"href","https://huggingface.co/docs/tokenizers"),c($,"rel","nofollow"),c(b,"id","carregando-diretamente-de-um-objeto-tokenizer"),c(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(b,"href","#carregando-diretamente-de-um-objeto-tokenizer"),c(v,"class","relative group"),c(C,"href","main_classes/tokenizer"),c(g,"id","carregando-de-um-arquivo-json"),c(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(g,"href","#carregando-de-um-arquivo-json"),c(y,"class","relative group"),c(W,"href","main_classes/tokenizer")},m(e,s){o(document.head,k),p(e,te,s),p(e,z,s),o(z,j),o(j,O),A(_,O,null),o(z,$e),o(z,x),o(x,Ee),p(e,re,s),p(e,f,s),o(f,Ue),o(f,H),o(H,we),o(f,Ve),o(f,$),o($,Je),o(f,Ze),p(e,le,s),p(e,I,s),o(I,qe),p(e,ne,s),A(E,e,s),p(e,ie,s),p(e,S,s),o(S,Ne),p(e,pe,s),p(e,v,s),o(v,b),o(b,Y),A(U,Y,null),o(v,Ae),o(v,K),o(K,Be),p(e,me,s),p(e,u,s),o(u,Fe),o(u,L),o(L,Pe),o(u,Ie),o(u,D),o(D,Se),o(u,Ce),p(e,de,s),A(w,e,s),p(e,ce,s),p(e,M,s),o(M,Qe),o(M,C),o(C,We),o(M,Ge),p(e,fe,s),p(e,y,s),o(y,g),o(g,ee),A(V,ee,null),o(y,Xe),o(y,ae),o(ae,Re),p(e,ue,s),p(e,Q,s),o(Q,Oe),p(e,he,s),A(J,e,s),p(e,ke,s),p(e,h,s),o(h,xe),o(h,oe),o(oe,He),o(h,Ye),o(h,se),o(se,Ke),o(h,Le),p(e,ze,s),A(Z,e,s),p(e,ve,s),p(e,T,s),o(T,De),o(T,W),o(W,ea),o(T,aa),ye=!0},p:Ua,i(e){ye||(B(_.$$.fragment,e),B(E.$$.fragment,e),B(U.$$.fragment,e),B(w.$$.fragment,e),B(V.$$.fragment,e),B(J.$$.fragment,e),B(Z.$$.fragment,e),ye=!0)},o(e){F(_.$$.fragment,e),F(E.$$.fragment,e),F(U.$$.fragment,e),F(w.$$.fragment,e),F(V.$$.fragment,e),F(J.$$.fragment,e),F(Z.$$.fragment,e),ye=!1},d(e){a(k),e&&a(te),e&&a(z),P(_),e&&a(re),e&&a(f),e&&a(le),e&&a(I),e&&a(ne),P(E,e),e&&a(ie),e&&a(S),e&&a(pe),e&&a(v),P(U),e&&a(me),e&&a(u),e&&a(de),P(w,e),e&&a(ce),e&&a(M),e&&a(fe),e&&a(y),P(V),e&&a(ue),e&&a(Q),e&&a(he),P(J,e),e&&a(ke),e&&a(h),e&&a(ze),P(Z,e),e&&a(ve),e&&a(T)}}}const Ja={local:"usando-os-tokenizers-do-tokenizers",sections:[{local:"carregando-diretamente-de-um-objeto-tokenizer",title:"Carregando diretamente de um objeto tokenizer"},{local:"carregando-de-um-arquivo-json",title:"Carregando de um arquivo JSON"}],title:"Usando os Tokenizers do \u{1F917} Tokenizers"};function Za(sa){return wa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ba extends Ta{constructor(k){super();_a(this,k,Za,Va,$a,{})}}export{Ba as default,Ja as metadata};
