import{S as fl,i as hl,s as jl,e as r,k as j,w as J,t as n,M as Ml,c as p,d as e,m as M,a as i,x as E,h as o,b as y,G as a,g as m,y as T,q as k,o as v,B as U,v as gl,L as ul}from"../../chunks/vendor-hf-doc-builder.js";import{T as he}from"../../chunks/Tip-hf-doc-builder.js";import{Y as ml}from"../../chunks/Youtube-hf-doc-builder.js";import{I as fe}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as as}from"../../chunks/CodeBlock-hf-doc-builder.js";import{F as dl,M as je}from"../../chunks/Markdown-hf-doc-builder.js";function bl(A){let t,u,l,f,b;return{c(){t=r("p"),u=n("Consulte a "),l=r("a"),f=n("p\xE1gina de tarefas de classifica\xE7\xE3o de tokens"),b=n(" para obter mais informa\xE7\xF5es sobre outras formas de classifica\xE7\xE3o de tokens e seus modelos, conjuntos de dados e m\xE9tricas associadas."),this.h()},l(g){t=p(g,"P",{});var w=i(t);u=o(w,"Consulte a "),l=p(w,"A",{href:!0,rel:!0});var x=i(l);f=o(x,"p\xE1gina de tarefas de classifica\xE7\xE3o de tokens"),x.forEach(e),b=o(w," para obter mais informa\xE7\xF5es sobre outras formas de classifica\xE7\xE3o de tokens e seus modelos, conjuntos de dados e m\xE9tricas associadas."),w.forEach(e),this.h()},h(){y(l,"href","https://huggingface.co/tasks/token-classification"),y(l,"rel","nofollow")},m(g,w){m(g,t,w),a(t,u),a(t,l),a(l,f),a(t,b)},d(g){g&&e(t)}}}function yl(A){let t,u;return t=new as({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`}}),{c(){J(t.$$.fragment)},l(l){E(t.$$.fragment,l)},m(l,f){T(t,l,f),u=!0},p:ul,i(l){u||(k(t.$$.fragment,l),u=!0)},o(l){v(t.$$.fragment,l),u=!1},d(l){U(t,l)}}}function wl(A){let t,u;return t=new je({props:{$$slots:{default:[yl]},$$scope:{ctx:A}}}),{c(){J(t.$$.fragment)},l(l){E(t.$$.fragment,l)},m(l,f){T(t,l,f),u=!0},p(l,f){const b={};f&2&&(b.$$scope={dirty:f,ctx:l}),t.$set(b)},i(l){u||(k(t.$$.fragment,l),u=!0)},o(l){v(t.$$.fragment,l),u=!1},d(l){U(t,l)}}}function $l(A){let t,u;return t=new as({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){J(t.$$.fragment)},l(l){E(t.$$.fragment,l)},m(l,f){T(t,l,f),u=!0},p:ul,i(l){u||(k(t.$$.fragment,l),u=!0)},o(l){v(t.$$.fragment,l),u=!1},d(l){U(t,l)}}}function _l(A){let t,u;return t=new je({props:{$$slots:{default:[$l]},$$scope:{ctx:A}}}),{c(){J(t.$$.fragment)},l(l){E(t.$$.fragment,l)},m(l,f){T(t,l,f),u=!0},p(l,f){const b={};f&2&&(b.$$scope={dirty:f,ctx:l}),t.$set(b)},i(l){u||(k(t.$$.fragment,l),u=!0)},o(l){v(t.$$.fragment,l),u=!1},d(l){U(t,l)}}}function Jl(A){let t,u,l,f,b,g,w,x;return{c(){t=r("p"),u=n("Se voc\xEA n\xE3o estiver familiarizado com o fine-tuning de um modelo com o "),l=r("code"),f=n("Trainer"),b=n(", d\xEA uma olhada no tutorial b\xE1sico "),g=r("a"),w=n("aqui"),x=n("!"),this.h()},l(C){t=p(C,"P",{});var _=i(t);u=o(_,"Se voc\xEA n\xE3o estiver familiarizado com o fine-tuning de um modelo com o "),l=p(_,"CODE",{});var Z=i(l);f=o(Z,"Trainer"),Z.forEach(e),b=o(_,", d\xEA uma olhada no tutorial b\xE1sico "),g=p(_,"A",{href:!0});var V=i(g);w=o(V,"aqui"),V.forEach(e),x=o(_,"!"),_.forEach(e),this.h()},h(){y(g,"href","../training#finetune-with-trainer")},m(C,_){m(C,t,_),a(t,u),a(t,l),a(l,f),a(t,b),a(t,g),a(g,w),a(t,x)},d(C){C&&e(t)}}}function El(A){let t,u,l,f,b,g,w,x,C,_,Z,V,P,z,Q,R,ns,q,gs,us,G,fs,Y,hs,N,js,W,S,B,O,L,os,F,es;return w=new as({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMkMlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMFRyYWluZXIlMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkJTIyJTJDJTIwbnVtX2xhYmVscyUzRDE0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">14</span>)`}}),C=new he({props:{$$slots:{default:[Jl]},$$scope:{ctx:A}}}),F=new as({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjIuJTJGcmVzdWx0cyUyMiUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX2V2YWxfYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX2Vwb2NocyUzRDMlMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRfZGVjYXklM0QwLjAxJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0R0b2tlbml6ZWRfd251dCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX3dudXQlNUIlMjJ0ZXN0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVyJTNEdG9rZW5pemVyJTJDJTBBJTIwJTIwJTIwJTIwZGF0YV9jb2xsYXRvciUzRGRhdGFfY29sbGF0b3IlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),{c(){t=r("p"),u=n("Carregue o DistilBERT com o "),l=r("code"),f=n("AutoModelForTokenClassification"),b=n(" junto com o n\xFAmero de r\xF3tulos esperados:"),g=j(),J(w.$$.fragment),x=j(),J(C.$$.fragment),_=j(),Z=r("p"),V=n("Nesse ponto, restam apenas tr\xEAs passos:"),P=j(),z=r("ol"),Q=r("li"),R=n("Definir seus hiperpar\xE2metros de treinamento em "),ns=r("code"),q=n("TrainingArguments"),gs=n("."),us=j(),G=r("li"),fs=n("Passar os argumentos de treinamento para o "),Y=r("code"),hs=n("Trainer"),N=n(" junto com o modelo, conjunto de dados, tokenizador e o data collator."),js=j(),W=r("li"),S=n("Chamar a fun\xE7\xE3o "),B=r("code"),O=n("train()"),L=n(" para executar o fine-tuning do seu modelo."),os=j(),J(F.$$.fragment)},l(h){t=p(h,"P",{});var I=i(t);u=o(I,"Carregue o DistilBERT com o "),l=p(I,"CODE",{});var ts=i(l);f=o(ts,"AutoModelForTokenClassification"),ts.forEach(e),b=o(I," junto com o n\xFAmero de r\xF3tulos esperados:"),I.forEach(e),g=M(h),E(w.$$.fragment,h),x=M(h),E(C.$$.fragment,h),_=M(h),Z=p(h,"P",{});var D=i(Z);V=o(D,"Nesse ponto, restam apenas tr\xEAs passos:"),D.forEach(e),P=M(h),z=p(h,"OL",{});var K=i(z);Q=p(K,"LI",{});var X=i(Q);R=o(X,"Definir seus hiperpar\xE2metros de treinamento em "),ns=p(X,"CODE",{});var ys=i(ns);q=o(ys,"TrainingArguments"),ys.forEach(e),gs=o(X,"."),X.forEach(e),us=M(K),G=p(K,"LI",{});var H=i(G);fs=o(H,"Passar os argumentos de treinamento para o "),Y=p(H,"CODE",{});var ss=i(Y);hs=o(ss,"Trainer"),ss.forEach(e),N=o(H," junto com o modelo, conjunto de dados, tokenizador e o data collator."),H.forEach(e),js=M(K),W=p(K,"LI",{});var ps=i(W);S=o(ps,"Chamar a fun\xE7\xE3o "),B=p(ps,"CODE",{});var ls=i(B);O=o(ls,"train()"),ls.forEach(e),L=o(ps," para executar o fine-tuning do seu modelo."),ps.forEach(e),K.forEach(e),os=M(h),E(F.$$.fragment,h)},m(h,I){m(h,t,I),a(t,u),a(t,l),a(l,f),a(t,b),m(h,g,I),T(w,h,I),m(h,x,I),T(C,h,I),m(h,_,I),m(h,Z,I),a(Z,V),m(h,P,I),m(h,z,I),a(z,Q),a(Q,R),a(Q,ns),a(ns,q),a(Q,gs),a(z,us),a(z,G),a(G,fs),a(G,Y),a(Y,hs),a(G,N),a(z,js),a(z,W),a(W,S),a(W,B),a(B,O),a(W,L),m(h,os,I),T(F,h,I),es=!0},p(h,I){const ts={};I&2&&(ts.$$scope={dirty:I,ctx:h}),C.$set(ts)},i(h){es||(k(w.$$.fragment,h),k(C.$$.fragment,h),k(F.$$.fragment,h),es=!0)},o(h){v(w.$$.fragment,h),v(C.$$.fragment,h),v(F.$$.fragment,h),es=!1},d(h){h&&e(t),h&&e(g),U(w,h),h&&e(x),U(C,h),h&&e(_),h&&e(Z),h&&e(P),h&&e(z),h&&e(os),U(F,h)}}}function Tl(A){let t,u;return t=new je({props:{$$slots:{default:[El]},$$scope:{ctx:A}}}),{c(){J(t.$$.fragment)},l(l){E(t.$$.fragment,l)},m(l,f){T(t,l,f),u=!0},p(l,f){const b={};f&2&&(b.$$scope={dirty:f,ctx:l}),t.$set(b)},i(l){u||(k(t.$$.fragment,l),u=!0)},o(l){v(t.$$.fragment,l),u=!1},d(l){U(t,l)}}}function kl(A){let t,u,l,f,b;return{c(){t=r("p"),u=n("Se voc\xEA n\xE3o estiver familiarizado com o fine-tuning de um modelo com o Keras, d\xEA uma olhada no tutorial b\xE1sico "),l=r("a"),f=n("aqui"),b=n("!"),this.h()},l(g){t=p(g,"P",{});var w=i(t);u=o(w,"Se voc\xEA n\xE3o estiver familiarizado com o fine-tuning de um modelo com o Keras, d\xEA uma olhada no tutorial b\xE1sico "),l=p(w,"A",{href:!0});var x=i(l);f=o(x,"aqui"),x.forEach(e),b=o(w,"!"),w.forEach(e),this.h()},h(){y(l,"href","training#finetune-with-keras")},m(g,w){m(g,t,w),a(t,u),a(t,l),a(l,f),a(t,b)},d(g){g&&e(t)}}}function vl(A){let t,u,l,f,b,g,w,x,C,_,Z,V,P,z,Q,R,ns,q,gs,us,G,fs,Y,hs,N,js,W,S,B,O,L,os,F,es,h,I,ts,D,K,X,ys,H,ss,ps,ls,xs,is,Cs;return z=new as({props:{code:"dGZfdHJhaW5fc2V0JTIwJTNEJTIwdG9rZW5pemVkX3dudXQlNUIlMjJ0cmFpbiUyMiU1RC50b190Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMGNvbHVtbnMlM0QlNUIlMjJhdHRlbnRpb25fbWFzayUyMiUyQyUyMCUyMmlucHV0X2lkcyUyMiUyQyUyMCUyMmxhYmVscyUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwY29sbGF0ZV9mbiUzRGRhdGFfY29sbGF0b3IlMkMlMEEpJTBBJTBBdGZfdmFsaWRhdGlvbl9zZXQlMjAlM0QlMjB0b2tlbml6ZWRfd251dCU1QiUyMnZhbGlkYXRpb24lMjIlNUQudG9fdGZfZGF0YXNldCglMEElMjAlMjAlMjAlMjBjb2x1bW5zJTNEJTVCJTIyYXR0ZW50aW9uX21hc2slMjIlMkMlMjAlMjJpbnB1dF9pZHMlMjIlMkMlMjAlMjJsYWJlbHMlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBzaHVmZmxlJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBjb2xsYXRlX2ZuJTNEZGF0YV9jb2xsYXRvciUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),R=new he({props:{$$slots:{default:[kl]},$$scope:{ctx:A}}}),G=new as({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEElMEFiYXRjaF9zaXplJTIwJTNEJTIwMTYlMEFudW1fdHJhaW5fZXBvY2hzJTIwJTNEJTIwMyUwQW51bV90cmFpbl9zdGVwcyUyMCUzRCUyMChsZW4odG9rZW5pemVkX3dudXQlNUIlMjJ0cmFpbiUyMiU1RCklMjAlMkYlMkYlMjBiYXRjaF9zaXplKSUyMColMjBudW1fdHJhaW5fZXBvY2hzJTBBb3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGUlMjAlM0QlMjBjcmVhdGVfb3B0aW1pemVyKCUwQSUyMCUyMCUyMCUyMGluaXRfbHIlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX3N0ZXBzJTNEbnVtX3RyYWluX3N0ZXBzJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5X3JhdGUlM0QwLjAxJTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, lr_schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_steps=num_train_steps,
<span class="hljs-meta">... </span>    weight_decay_rate=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>)`}}),B=new as({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMiUyQyUyMG51bV9sYWJlbHMlM0QyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">2</span>)`}}),D=new as({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplcik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),is=new as({props:{code:"bW9kZWwuZml0KHglM0R0Zl90cmFpbl9zZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl92YWxpZGF0aW9uX3NldCUyQyUyMGVwb2NocyUzRDMp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>)'}}),{c(){t=r("p"),u=n("Para executar o fine-tuning de um modelo no TensorFlow, comece convertendo seu conjunto de dados para o formato "),l=r("code"),f=n("tf.data.Dataset"),b=n(" com "),g=r("a"),w=r("code"),x=n("to_tf_dataset"),C=n(". Nessa execu\xE7\xE3o voc\xEA dever\xE1 especificar as entradas e r\xF3tulos (no par\xE2metro "),_=r("code"),Z=n("columns"),V=n("), se deseja embaralhar o conjunto de dados, o tamanho do batch e o data collator:"),P=j(),J(z.$$.fragment),Q=j(),J(R.$$.fragment),ns=j(),q=r("p"),gs=n("Configure o otimizador e alguns hiperpar\xE2metros de treinamento:"),us=j(),J(G.$$.fragment),fs=j(),Y=r("p"),hs=n("Carregue o DistilBERT com o "),N=r("code"),js=n("TFAutoModelForTokenClassification"),W=n(" junto com o n\xFAmero de r\xF3tulos esperados:"),S=j(),J(B.$$.fragment),O=j(),L=r("p"),os=n("Configure o modelo para treinamento com o m\xE9todo "),F=r("a"),es=r("code"),h=n("compile"),I=n(":"),ts=j(),J(D.$$.fragment),K=j(),X=r("p"),ys=n("Chame o m\xE9todo "),H=r("a"),ss=r("code"),ps=n("fit"),ls=n(" para executar o fine-tuning do modelo:"),xs=j(),J(is.$$.fragment),this.h()},l(c){t=p(c,"P",{});var $=i(t);u=o($,"Para executar o fine-tuning de um modelo no TensorFlow, comece convertendo seu conjunto de dados para o formato "),l=p($,"CODE",{});var cs=i(l);f=o(cs,"tf.data.Dataset"),cs.forEach(e),b=o($," com "),g=p($,"A",{href:!0,rel:!0});var qs=i(g);w=p(qs,"CODE",{});var ms=i(w);x=o(ms,"to_tf_dataset"),ms.forEach(e),qs.forEach(e),C=o($,". Nessa execu\xE7\xE3o voc\xEA dever\xE1 especificar as entradas e r\xF3tulos (no par\xE2metro "),_=p($,"CODE",{});var ta=i(_);Z=o(ta,"columns"),ta.forEach(e),V=o($,"), se deseja embaralhar o conjunto de dados, o tamanho do batch e o data collator:"),$.forEach(e),P=M(c),E(z.$$.fragment,c),Q=M(c),E(R.$$.fragment,c),ns=M(c),q=p(c,"P",{});var Is=i(q);gs=o(Is,"Configure o otimizador e alguns hiperpar\xE2metros de treinamento:"),Is.forEach(e),us=M(c),E(G.$$.fragment,c),fs=M(c),Y=p(c,"P",{});var As=i(Y);hs=o(As,"Carregue o DistilBERT com o "),N=p(As,"CODE",{});var la=i(N);js=o(la,"TFAutoModelForTokenClassification"),la.forEach(e),W=o(As," junto com o n\xFAmero de r\xF3tulos esperados:"),As.forEach(e),S=M(c),E(B.$$.fragment,c),O=M(c),L=p(c,"P",{});var ws=i(L);os=o(ws,"Configure o modelo para treinamento com o m\xE9todo "),F=p(ws,"A",{href:!0,rel:!0});var na=i(F);es=p(na,"CODE",{});var oa=i(es);h=o(oa,"compile"),oa.forEach(e),na.forEach(e),I=o(ws,":"),ws.forEach(e),ts=M(c),E(D.$$.fragment,c),K=M(c),X=p(c,"P",{});var Ts=i(X);ys=o(Ts,"Chame o m\xE9todo "),H=p(Ts,"A",{href:!0,rel:!0});var ds=i(H);ss=p(ds,"CODE",{});var ks=i(ss);ps=o(ks,"fit"),ks.forEach(e),ds.forEach(e),ls=o(Ts," para executar o fine-tuning do modelo:"),Ts.forEach(e),xs=M(c),E(is.$$.fragment,c),this.h()},h(){y(g,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),y(g,"rel","nofollow"),y(F,"href","https://keras.io/api/models/model_training_apis/#compile-method"),y(F,"rel","nofollow"),y(H,"href","https://keras.io/api/models/model_training_apis/#fit-method"),y(H,"rel","nofollow")},m(c,$){m(c,t,$),a(t,u),a(t,l),a(l,f),a(t,b),a(t,g),a(g,w),a(w,x),a(t,C),a(t,_),a(_,Z),a(t,V),m(c,P,$),T(z,c,$),m(c,Q,$),T(R,c,$),m(c,ns,$),m(c,q,$),a(q,gs),m(c,us,$),T(G,c,$),m(c,fs,$),m(c,Y,$),a(Y,hs),a(Y,N),a(N,js),a(Y,W),m(c,S,$),T(B,c,$),m(c,O,$),m(c,L,$),a(L,os),a(L,F),a(F,es),a(es,h),a(L,I),m(c,ts,$),T(D,c,$),m(c,K,$),m(c,X,$),a(X,ys),a(X,H),a(H,ss),a(ss,ps),a(X,ls),m(c,xs,$),T(is,c,$),Cs=!0},p(c,$){const cs={};$&2&&(cs.$$scope={dirty:$,ctx:c}),R.$set(cs)},i(c){Cs||(k(z.$$.fragment,c),k(R.$$.fragment,c),k(G.$$.fragment,c),k(B.$$.fragment,c),k(D.$$.fragment,c),k(is.$$.fragment,c),Cs=!0)},o(c){v(z.$$.fragment,c),v(R.$$.fragment,c),v(G.$$.fragment,c),v(B.$$.fragment,c),v(D.$$.fragment,c),v(is.$$.fragment,c),Cs=!1},d(c){c&&e(t),c&&e(P),U(z,c),c&&e(Q),U(R,c),c&&e(ns),c&&e(q),c&&e(us),U(G,c),c&&e(fs),c&&e(Y),c&&e(S),U(B,c),c&&e(O),c&&e(L),c&&e(ts),U(D,c),c&&e(K),c&&e(X),c&&e(xs),U(is,c)}}}function Ul(A){let t,u;return t=new je({props:{$$slots:{default:[vl]},$$scope:{ctx:A}}}),{c(){J(t.$$.fragment)},l(l){E(t.$$.fragment,l)},m(l,f){T(t,l,f),u=!0},p(l,f){const b={};f&2&&(b.$$scope={dirty:f,ctx:l}),t.$set(b)},i(l){u||(k(t.$$.fragment,l),u=!0)},o(l){v(t.$$.fragment,l),u=!1},d(l){U(t,l)}}}function xl(A){let t,u,l,f,b,g,w,x;return{c(){t=r("p"),u=n("Para obter um exemplo mais aprofundado de como executar o fine-tuning de um modelo para classifica\xE7\xE3o de tokens, d\xEA uma olhada nesse "),l=r("a"),f=n("notebook utilizando PyTorch"),b=n(" ou nesse "),g=r("a"),w=n("notebook utilizando TensorFlow"),x=n("."),this.h()},l(C){t=p(C,"P",{});var _=i(t);u=o(_,"Para obter um exemplo mais aprofundado de como executar o fine-tuning de um modelo para classifica\xE7\xE3o de tokens, d\xEA uma olhada nesse "),l=p(_,"A",{href:!0,rel:!0});var Z=i(l);f=o(Z,"notebook utilizando PyTorch"),Z.forEach(e),b=o(_," ou nesse "),g=p(_,"A",{href:!0,rel:!0});var V=i(g);w=o(V,"notebook utilizando TensorFlow"),V.forEach(e),x=o(_,"."),_.forEach(e),this.h()},h(){y(l,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb"),y(l,"rel","nofollow"),y(g,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb"),y(g,"rel","nofollow")},m(C,_){m(C,t,_),a(t,u),a(t,l),a(l,f),a(t,b),a(t,g),a(g,w),a(t,x)},d(C){C&&e(t)}}}function Cl(A){let t,u,l,f,b,g,w,x,C,_,Z,V,P,z,Q,R,ns,q,gs,us,G,fs,Y,hs,N,js,W,S,B,O,L,os,F,es,h,I,ts,D,K,X,ys,H,ss,ps,ls,xs,is,Cs,c,$,cs,qs,ms,ta,Is,As,la,ws,na,oa,Ts,ds,ks,fa,Me,ge,be,$s,ha,ye,we,ja,$e,_e,Ma,Je,Ee,Te,ra,ga,ke,ve,Va,vs,Zs,ba,Ys,Ue,ya,xe,qa,Ns,Ya,Rs,Ce,wa,Ie,Ae,Na,Ds,Da,Gs,Ze,$a,Re,Ge,Qa,Qs,Sa,_s,ze,_a,We,Xe,Ja,Be,Fe,Ha,Js,Ss,Ve,Hs,Ea,qe,Ye,Ne,bs,De,Ta,Qe,Se,ka,He,Pe,va,Oe,Le,Ke,Ps,st,Ua,at,et,Pa,pa,tt,Oa,Os,La,Ms,lt,Ls,xa,nt,ot,Ca,rt,pt,Ia,it,ct,Ka,Ks,se,rs,mt,Aa,dt,ut,Za,ft,ht,Ra,jt,Mt,Ga,gt,bt,ae,zs,ee,Us,Ws,za,sa,yt,Wa,wt,te,Xs,le,Bs,ne;return g=new fe({}),Z=new ml({props:{id:"wVHdVlPScxA"}}),N=new he({props:{$$slots:{default:[bl]},$$scope:{ctx:A}}}),O=new fe({}),D=new as({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBd251dCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ3bnV0XzE3JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`}}),ss=new as({props:{code:"d251dCU1QiUyMnRyYWluJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`}}),cs=new as({props:{code:"bGFiZWxfbGlzdCUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RC5mZWF0dXJlcyU1QmYlMjJuZXJfdGFncyUyMiU1RC5mZWF0dXJlLm5hbWVzJTBBbGFiZWxfbGlzdA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`}}),Ys=new fe({}),Ns=new ml({props:{id:"iY2AZYdZAr0"}}),Ds=new as({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),Qs=new as({props:{code:"dG9rZW5pemVkX2lucHV0JTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGUlNUIlMjJ0b2tlbnMlMjIlNUQlMkMlMjBpc19zcGxpdF9pbnRvX3dvcmRzJTNEVHJ1ZSklMEF0b2tlbnMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF9pZHNfdG9fdG9rZW5zKHRva2VuaXplZF9pbnB1dCU1QiUyMmlucHV0X2lkcyUyMiU1RCklMEF0b2tlbnM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),Os=new as({props:{code:"ZGVmJTIwdG9rZW5pemVfYW5kX2FsaWduX2xhYmVscyhleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydG9rZW5zJTIyJTVEJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMjBpc19zcGxpdF9pbnRvX3dvcmRzJTNEVHJ1ZSklMEElMEElMjAlMjAlMjAlMjBsYWJlbHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBpJTJDJTIwbGFiZWwlMjBpbiUyMGVudW1lcmF0ZShleGFtcGxlcyU1QmYlMjJuZXJfdGFncyUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkcyUyMCUzRCUyMHRva2VuaXplZF9pbnB1dHMud29yZF9pZHMoYmF0Y2hfaW5kZXglM0RpKSUyMCUyMCUyMyUyME1hcCUyMHRva2VucyUyMHRvJTIwdGhlaXIlMjByZXNwZWN0aXZlJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyME5vbmUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjB3b3JkX2lkeCUyMGluJTIwd29yZF9pZHMlM0ElMjAlMjAlMjMlMjBTZXQlMjB0aGUlMjBzcGVjaWFsJTIwdG9rZW5zJTIwdG8lMjAtMTAwLiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZHglMjBpcyUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKC0xMDApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxpZiUyMHdvcmRfaWR4JTIwISUzRCUyMHByZXZpb3VzX3dvcmRfaWR4JTNBJTIwJTIwJTIzJTIwT25seSUyMGxhYmVsJTIwdGhlJTIwZmlyc3QlMjB0b2tlbiUyMG9mJTIwYSUyMGdpdmVuJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKGxhYmVsJTVCd29yZF9pZHglNUQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVsX2lkcy5hcHBlbmQoLTEwMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyMHdvcmRfaWR4JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFiZWxzLmFwcGVuZChsYWJlbF9pZHMpJTBBJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVkX2lucHV0cyU1QiUyMmxhYmVscyUyMiU1RCUyMCUzRCUyMGxhYmVscyUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHRva2VuaXplZF9pbnB1dHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
<span class="hljs-meta">... </span>        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
<span class="hljs-meta">... </span>        previous_word_idx = <span class="hljs-literal">None</span>
<span class="hljs-meta">... </span>        label_ids = []
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
<span class="hljs-meta">... </span>                label_ids.append(label[word_idx])
<span class="hljs-meta">... </span>            <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            previous_word_idx = word_idx
<span class="hljs-meta">... </span>        labels.append(label_ids)

<span class="hljs-meta">... </span>    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenized_inputs`}}),Ks=new as({props:{code:"dG9rZW5pemVkX3dudXQlMjAlM0QlMjB3bnV0Lm1hcCh0b2tlbml6ZV9hbmRfYWxpZ25fbGFiZWxzJTJDJTIwYmF0Y2hlZCUzRFRydWUp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)'}}),zs=new dl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[_l],pytorch:[wl]},$$scope:{ctx:A}}}),sa=new fe({}),Xs=new dl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Ul],pytorch:[Tl]},$$scope:{ctx:A}}}),Bs=new he({props:{$$slots:{default:[xl]},$$scope:{ctx:A}}}),{c(){t=r("meta"),u=j(),l=r("h1"),f=r("a"),b=r("span"),J(g.$$.fragment),w=j(),x=r("span"),C=n("Classifica\xE7\xE3o de tokens"),_=j(),J(Z.$$.fragment),V=j(),P=r("p"),z=n("A classifica\xE7\xE3o de tokens atribui um r\xF3tulo a tokens individuais em uma frase. Uma das tarefas de classifica\xE7\xE3o de tokens mais comuns \xE9 o Reconhecimento de Entidade Nomeada, tamb\xE9m chamada de NER (sigla em ingl\xEAs para Named Entity Recognition). O NER tenta encontrar um r\xF3tulo para cada entidade em uma frase, como uma pessoa, local ou organiza\xE7\xE3o."),Q=j(),R=r("p"),ns=n("Este guia mostrar\xE1 como realizar o fine-tuning do "),q=r("a"),gs=n("DistilBERT"),us=n(" no conjunto de dados "),G=r("a"),fs=n("WNUT 17"),Y=n(" para detectar novas entidades."),hs=j(),J(N.$$.fragment),js=j(),W=r("h2"),S=r("a"),B=r("span"),J(O.$$.fragment),L=j(),os=r("span"),F=n("Carregando o conjunto de dados WNUT 17"),es=j(),h=r("p"),I=n("Carregue o conjunto de dados WNUT 17 da biblioteca \u{1F917} Datasets:"),ts=j(),J(D.$$.fragment),K=j(),X=r("p"),ys=n("E d\xEA uma olhada em um exemplo:"),H=j(),J(ss.$$.fragment),ps=j(),ls=r("p"),xs=n("Cada n\xFAmero em "),is=r("code"),Cs=n("ner_tags"),c=n(" representa uma entidade. Converta o n\xFAmero em um r\xF3tulo para obter mais informa\xE7\xF5es:"),$=j(),J(cs.$$.fragment),qs=j(),ms=r("p"),ta=n("O "),Is=r("code"),As=n("ner_tag"),la=n(" descreve uma entidade, como uma organiza\xE7\xE3o, local ou pessoa. A letra que prefixa cada "),ws=r("code"),na=n("ner_tag"),oa=n(" indica a posi\xE7\xE3o do token da entidade:"),Ts=j(),ds=r("ul"),ks=r("li"),fa=r("code"),Me=n("B-"),ge=n(" indica o in\xEDcio de uma entidade."),be=j(),$s=r("li"),ha=r("code"),ye=n("I-"),we=n(" indica que um token est\xE1 contido dentro da mesma entidade (por exemplo, o token "),ja=r("code"),$e=n("State"),_e=n(" pode fazer parte de uma entidade como "),Ma=r("code"),Je=n("Empire State Building"),Ee=n(")."),Te=j(),ra=r("li"),ga=r("code"),ke=n("0"),ve=n(" indica que o token n\xE3o corresponde a nenhuma entidade."),Va=j(),vs=r("h2"),Zs=r("a"),ba=r("span"),J(Ys.$$.fragment),Ue=j(),ya=r("span"),xe=n("Pr\xE9-processamento"),qa=j(),J(Ns.$$.fragment),Ya=j(),Rs=r("p"),Ce=n("Carregue o tokenizer do DistilBERT para processar os "),wa=r("code"),Ie=n("tokens"),Ae=n(":"),Na=j(),J(Ds.$$.fragment),Da=j(),Gs=r("p"),Ze=n("Como a entrada j\xE1 foi dividida em palavras, defina "),$a=r("code"),Re=n("is_split_into_words=True"),Ge=n(" para tokenizar as palavras em subpalavras:"),Qa=j(),J(Qs.$$.fragment),Sa=j(),_s=r("p"),ze=n("Ao adicionar os tokens especiais "),_a=r("code"),We=n("[CLS]"),Xe=n(" e "),Ja=r("code"),Be=n("[SEP]"),Fe=n(" e a tokeniza\xE7\xE3o de subpalavras uma incompatibilidade \xE9 gerada entre a entrada e os r\xF3tulos. Uma \xFAnica palavra correspondente a um \xFAnico r\xF3tulo pode ser dividida em duas subpalavras. Voc\xEA precisar\xE1 realinhar os tokens e os r\xF3tulos da seguinte forma:"),Ha=j(),Js=r("ol"),Ss=r("li"),Ve=n("Mapeie todos os tokens para a palavra correspondente com o m\xE9todo "),Hs=r("a"),Ea=r("code"),qe=n("word_ids"),Ye=n("."),Ne=j(),bs=r("li"),De=n("Atribuindo o r\xF3tulo "),Ta=r("code"),Qe=n("-100"),Se=n(" aos tokens especiais "),ka=r("code"),He=n("[CLS]"),Pe=n(" e "),va=r("code"),Oe=n("[SEP]"),Le=n(" para que a fun\xE7\xE3o de loss do PyTorch ignore eles."),Ke=j(),Ps=r("li"),st=n("Rotular apenas o primeiro token de uma determinada palavra. Atribuindo "),Ua=r("code"),at=n("-100"),et=n(" a outros subtokens da mesma palavra."),Pa=j(),pa=r("p"),tt=n("Aqui est\xE1 como voc\xEA pode criar uma fun\xE7\xE3o para realinhar os tokens e r\xF3tulos e truncar sequ\xEAncias para n\xE3o serem maiores que o comprimento m\xE1ximo de entrada do DistilBERT:"),Oa=j(),J(Os.$$.fragment),La=j(),Ms=r("p"),lt=n("Use a fun\xE7\xE3o "),Ls=r("a"),xa=r("code"),nt=n("map"),ot=n(" do \u{1F917} Datasets para tokenizar e alinhar os r\xF3tulos em todo o conjunto de dados. Voc\xEA pode acelerar a fun\xE7\xE3o "),Ca=r("code"),rt=n("map"),pt=n(" configurando "),Ia=r("code"),it=n("batched=True"),ct=n(" para processar v\xE1rios elementos do conjunto de dados de uma s\xF3 vez:"),Ka=j(),J(Ks.$$.fragment),se=j(),rs=r("p"),mt=n("Use o "),Aa=r("code"),dt=n("DataCollatorForTokenClassification"),ut=n(" para criar um batch de exemplos. Ele tamb\xE9m "),Za=r("em"),ft=n("preencher\xE1 dinamicamente"),ht=n(" seu texto e r\xF3tulos para o comprimento do elemento mais longo em seu batch, para que tenham um comprimento uniforme. Embora seja poss\xEDvel preencher seu texto na fun\xE7\xE3o "),Ra=r("code"),jt=n("tokenizer"),Mt=n(" configurando "),Ga=r("code"),gt=n("padding=True"),bt=n(", o preenchimento din\xE2mico \xE9 mais eficiente."),ae=j(),J(zs.$$.fragment),ee=j(),Us=r("h2"),Ws=r("a"),za=r("span"),J(sa.$$.fragment),yt=j(),Wa=r("span"),wt=n("Treinamento"),te=j(),J(Xs.$$.fragment),le=j(),J(Bs.$$.fragment),this.h()},l(s){const d=Ml('[data-svelte="svelte-1phssyn"]',document.head);t=p(d,"META",{name:!0,content:!0}),d.forEach(e),u=M(s),l=p(s,"H1",{class:!0});var aa=i(l);f=p(aa,"A",{id:!0,class:!0,href:!0});var Xa=i(f);b=p(Xa,"SPAN",{});var Ba=i(b);E(g.$$.fragment,Ba),Ba.forEach(e),Xa.forEach(e),w=M(aa),x=p(aa,"SPAN",{});var Fa=i(x);C=o(Fa,"Classifica\xE7\xE3o de tokens"),Fa.forEach(e),aa.forEach(e),_=M(s),E(Z.$$.fragment,s),V=M(s),P=p(s,"P",{});var Jt=i(P);z=o(Jt,"A classifica\xE7\xE3o de tokens atribui um r\xF3tulo a tokens individuais em uma frase. Uma das tarefas de classifica\xE7\xE3o de tokens mais comuns \xE9 o Reconhecimento de Entidade Nomeada, tamb\xE9m chamada de NER (sigla em ingl\xEAs para Named Entity Recognition). O NER tenta encontrar um r\xF3tulo para cada entidade em uma frase, como uma pessoa, local ou organiza\xE7\xE3o."),Jt.forEach(e),Q=M(s),R=p(s,"P",{});var ia=i(R);ns=o(ia,"Este guia mostrar\xE1 como realizar o fine-tuning do "),q=p(ia,"A",{href:!0,rel:!0});var Et=i(q);gs=o(Et,"DistilBERT"),Et.forEach(e),us=o(ia," no conjunto de dados "),G=p(ia,"A",{href:!0,rel:!0});var Tt=i(G);fs=o(Tt,"WNUT 17"),Tt.forEach(e),Y=o(ia," para detectar novas entidades."),ia.forEach(e),hs=M(s),E(N.$$.fragment,s),js=M(s),W=p(s,"H2",{class:!0});var oe=i(W);S=p(oe,"A",{id:!0,class:!0,href:!0});var kt=i(S);B=p(kt,"SPAN",{});var vt=i(B);E(O.$$.fragment,vt),vt.forEach(e),kt.forEach(e),L=M(oe),os=p(oe,"SPAN",{});var Ut=i(os);F=o(Ut,"Carregando o conjunto de dados WNUT 17"),Ut.forEach(e),oe.forEach(e),es=M(s),h=p(s,"P",{});var xt=i(h);I=o(xt,"Carregue o conjunto de dados WNUT 17 da biblioteca \u{1F917} Datasets:"),xt.forEach(e),ts=M(s),E(D.$$.fragment,s),K=M(s),X=p(s,"P",{});var Ct=i(X);ys=o(Ct,"E d\xEA uma olhada em um exemplo:"),Ct.forEach(e),H=M(s),E(ss.$$.fragment,s),ps=M(s),ls=p(s,"P",{});var re=i(ls);xs=o(re,"Cada n\xFAmero em "),is=p(re,"CODE",{});var It=i(is);Cs=o(It,"ner_tags"),It.forEach(e),c=o(re," representa uma entidade. Converta o n\xFAmero em um r\xF3tulo para obter mais informa\xE7\xF5es:"),re.forEach(e),$=M(s),E(cs.$$.fragment,s),qs=M(s),ms=p(s,"P",{});var ca=i(ms);ta=o(ca,"O "),Is=p(ca,"CODE",{});var At=i(Is);As=o(At,"ner_tag"),At.forEach(e),la=o(ca," descreve uma entidade, como uma organiza\xE7\xE3o, local ou pessoa. A letra que prefixa cada "),ws=p(ca,"CODE",{});var Zt=i(ws);na=o(Zt,"ner_tag"),Zt.forEach(e),oa=o(ca," indica a posi\xE7\xE3o do token da entidade:"),ca.forEach(e),Ts=M(s),ds=p(s,"UL",{});var ma=i(ds);ks=p(ma,"LI",{});var $t=i(ks);fa=p($t,"CODE",{});var Rt=i(fa);Me=o(Rt,"B-"),Rt.forEach(e),ge=o($t," indica o in\xEDcio de uma entidade."),$t.forEach(e),be=M(ma),$s=p(ma,"LI",{});var ea=i($s);ha=p(ea,"CODE",{});var Gt=i(ha);ye=o(Gt,"I-"),Gt.forEach(e),we=o(ea," indica que um token est\xE1 contido dentro da mesma entidade (por exemplo, o token "),ja=p(ea,"CODE",{});var zt=i(ja);$e=o(zt,"State"),zt.forEach(e),_e=o(ea," pode fazer parte de uma entidade como "),Ma=p(ea,"CODE",{});var Wt=i(Ma);Je=o(Wt,"Empire State Building"),Wt.forEach(e),Ee=o(ea,")."),ea.forEach(e),Te=M(ma),ra=p(ma,"LI",{});var _t=i(ra);ga=p(_t,"CODE",{});var Xt=i(ga);ke=o(Xt,"0"),Xt.forEach(e),ve=o(_t," indica que o token n\xE3o corresponde a nenhuma entidade."),_t.forEach(e),ma.forEach(e),Va=M(s),vs=p(s,"H2",{class:!0});var pe=i(vs);Zs=p(pe,"A",{id:!0,class:!0,href:!0});var Bt=i(Zs);ba=p(Bt,"SPAN",{});var Ft=i(ba);E(Ys.$$.fragment,Ft),Ft.forEach(e),Bt.forEach(e),Ue=M(pe),ya=p(pe,"SPAN",{});var Vt=i(ya);xe=o(Vt,"Pr\xE9-processamento"),Vt.forEach(e),pe.forEach(e),qa=M(s),E(Ns.$$.fragment,s),Ya=M(s),Rs=p(s,"P",{});var ie=i(Rs);Ce=o(ie,"Carregue o tokenizer do DistilBERT para processar os "),wa=p(ie,"CODE",{});var qt=i(wa);Ie=o(qt,"tokens"),qt.forEach(e),Ae=o(ie,":"),ie.forEach(e),Na=M(s),E(Ds.$$.fragment,s),Da=M(s),Gs=p(s,"P",{});var ce=i(Gs);Ze=o(ce,"Como a entrada j\xE1 foi dividida em palavras, defina "),$a=p(ce,"CODE",{});var Yt=i($a);Re=o(Yt,"is_split_into_words=True"),Yt.forEach(e),Ge=o(ce," para tokenizar as palavras em subpalavras:"),ce.forEach(e),Qa=M(s),E(Qs.$$.fragment,s),Sa=M(s),_s=p(s,"P",{});var da=i(_s);ze=o(da,"Ao adicionar os tokens especiais "),_a=p(da,"CODE",{});var Nt=i(_a);We=o(Nt,"[CLS]"),Nt.forEach(e),Xe=o(da," e "),Ja=p(da,"CODE",{});var Dt=i(Ja);Be=o(Dt,"[SEP]"),Dt.forEach(e),Fe=o(da," e a tokeniza\xE7\xE3o de subpalavras uma incompatibilidade \xE9 gerada entre a entrada e os r\xF3tulos. Uma \xFAnica palavra correspondente a um \xFAnico r\xF3tulo pode ser dividida em duas subpalavras. Voc\xEA precisar\xE1 realinhar os tokens e os r\xF3tulos da seguinte forma:"),da.forEach(e),Ha=M(s),Js=p(s,"OL",{});var ua=i(Js);Ss=p(ua,"LI",{});var me=i(Ss);Ve=o(me,"Mapeie todos os tokens para a palavra correspondente com o m\xE9todo "),Hs=p(me,"A",{href:!0,rel:!0});var Qt=i(Hs);Ea=p(Qt,"CODE",{});var St=i(Ea);qe=o(St,"word_ids"),St.forEach(e),Qt.forEach(e),Ye=o(me,"."),me.forEach(e),Ne=M(ua),bs=p(ua,"LI",{});var Fs=i(bs);De=o(Fs,"Atribuindo o r\xF3tulo "),Ta=p(Fs,"CODE",{});var Ht=i(Ta);Qe=o(Ht,"-100"),Ht.forEach(e),Se=o(Fs," aos tokens especiais "),ka=p(Fs,"CODE",{});var Pt=i(ka);He=o(Pt,"[CLS]"),Pt.forEach(e),Pe=o(Fs," e "),va=p(Fs,"CODE",{});var Ot=i(va);Oe=o(Ot,"[SEP]"),Ot.forEach(e),Le=o(Fs," para que a fun\xE7\xE3o de loss do PyTorch ignore eles."),Fs.forEach(e),Ke=M(ua),Ps=p(ua,"LI",{});var de=i(Ps);st=o(de,"Rotular apenas o primeiro token de uma determinada palavra. Atribuindo "),Ua=p(de,"CODE",{});var Lt=i(Ua);at=o(Lt,"-100"),Lt.forEach(e),et=o(de," a outros subtokens da mesma palavra."),de.forEach(e),ua.forEach(e),Pa=M(s),pa=p(s,"P",{});var Kt=i(pa);tt=o(Kt,"Aqui est\xE1 como voc\xEA pode criar uma fun\xE7\xE3o para realinhar os tokens e r\xF3tulos e truncar sequ\xEAncias para n\xE3o serem maiores que o comprimento m\xE1ximo de entrada do DistilBERT:"),Kt.forEach(e),Oa=M(s),E(Os.$$.fragment,s),La=M(s),Ms=p(s,"P",{});var Vs=i(Ms);lt=o(Vs,"Use a fun\xE7\xE3o "),Ls=p(Vs,"A",{href:!0,rel:!0});var sl=i(Ls);xa=p(sl,"CODE",{});var al=i(xa);nt=o(al,"map"),al.forEach(e),sl.forEach(e),ot=o(Vs," do \u{1F917} Datasets para tokenizar e alinhar os r\xF3tulos em todo o conjunto de dados. Voc\xEA pode acelerar a fun\xE7\xE3o "),Ca=p(Vs,"CODE",{});var el=i(Ca);rt=o(el,"map"),el.forEach(e),pt=o(Vs," configurando "),Ia=p(Vs,"CODE",{});var tl=i(Ia);it=o(tl,"batched=True"),tl.forEach(e),ct=o(Vs," para processar v\xE1rios elementos do conjunto de dados de uma s\xF3 vez:"),Vs.forEach(e),Ka=M(s),E(Ks.$$.fragment,s),se=M(s),rs=p(s,"P",{});var Es=i(rs);mt=o(Es,"Use o "),Aa=p(Es,"CODE",{});var ll=i(Aa);dt=o(ll,"DataCollatorForTokenClassification"),ll.forEach(e),ut=o(Es," para criar um batch de exemplos. Ele tamb\xE9m "),Za=p(Es,"EM",{});var nl=i(Za);ft=o(nl,"preencher\xE1 dinamicamente"),nl.forEach(e),ht=o(Es," seu texto e r\xF3tulos para o comprimento do elemento mais longo em seu batch, para que tenham um comprimento uniforme. Embora seja poss\xEDvel preencher seu texto na fun\xE7\xE3o "),Ra=p(Es,"CODE",{});var ol=i(Ra);jt=o(ol,"tokenizer"),ol.forEach(e),Mt=o(Es," configurando "),Ga=p(Es,"CODE",{});var rl=i(Ga);gt=o(rl,"padding=True"),rl.forEach(e),bt=o(Es,", o preenchimento din\xE2mico \xE9 mais eficiente."),Es.forEach(e),ae=M(s),E(zs.$$.fragment,s),ee=M(s),Us=p(s,"H2",{class:!0});var ue=i(Us);Ws=p(ue,"A",{id:!0,class:!0,href:!0});var pl=i(Ws);za=p(pl,"SPAN",{});var il=i(za);E(sa.$$.fragment,il),il.forEach(e),pl.forEach(e),yt=M(ue),Wa=p(ue,"SPAN",{});var cl=i(Wa);wt=o(cl,"Treinamento"),cl.forEach(e),ue.forEach(e),te=M(s),E(Xs.$$.fragment,s),le=M(s),E(Bs.$$.fragment,s),this.h()},h(){y(t,"name","hf:doc:metadata"),y(t,"content",JSON.stringify(Il)),y(f,"id","classificao-de-tokens"),y(f,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(f,"href","#classificao-de-tokens"),y(l,"class","relative group"),y(q,"href","https://huggingface.co/distilbert-base-uncased"),y(q,"rel","nofollow"),y(G,"href","https://huggingface.co/datasets/wnut_17"),y(G,"rel","nofollow"),y(S,"id","carregando-o-conjunto-de-dados-wnut-17"),y(S,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(S,"href","#carregando-o-conjunto-de-dados-wnut-17"),y(W,"class","relative group"),y(Zs,"id","prprocessamento"),y(Zs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(Zs,"href","#prprocessamento"),y(vs,"class","relative group"),y(Hs,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#tokenizers.Encoding.word_ids"),y(Hs,"rel","nofollow"),y(Ls,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),y(Ls,"rel","nofollow"),y(Ws,"id","treinamento"),y(Ws,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(Ws,"href","#treinamento"),y(Us,"class","relative group")},m(s,d){a(document.head,t),m(s,u,d),m(s,l,d),a(l,f),a(f,b),T(g,b,null),a(l,w),a(l,x),a(x,C),m(s,_,d),T(Z,s,d),m(s,V,d),m(s,P,d),a(P,z),m(s,Q,d),m(s,R,d),a(R,ns),a(R,q),a(q,gs),a(R,us),a(R,G),a(G,fs),a(R,Y),m(s,hs,d),T(N,s,d),m(s,js,d),m(s,W,d),a(W,S),a(S,B),T(O,B,null),a(W,L),a(W,os),a(os,F),m(s,es,d),m(s,h,d),a(h,I),m(s,ts,d),T(D,s,d),m(s,K,d),m(s,X,d),a(X,ys),m(s,H,d),T(ss,s,d),m(s,ps,d),m(s,ls,d),a(ls,xs),a(ls,is),a(is,Cs),a(ls,c),m(s,$,d),T(cs,s,d),m(s,qs,d),m(s,ms,d),a(ms,ta),a(ms,Is),a(Is,As),a(ms,la),a(ms,ws),a(ws,na),a(ms,oa),m(s,Ts,d),m(s,ds,d),a(ds,ks),a(ks,fa),a(fa,Me),a(ks,ge),a(ds,be),a(ds,$s),a($s,ha),a(ha,ye),a($s,we),a($s,ja),a(ja,$e),a($s,_e),a($s,Ma),a(Ma,Je),a($s,Ee),a(ds,Te),a(ds,ra),a(ra,ga),a(ga,ke),a(ra,ve),m(s,Va,d),m(s,vs,d),a(vs,Zs),a(Zs,ba),T(Ys,ba,null),a(vs,Ue),a(vs,ya),a(ya,xe),m(s,qa,d),T(Ns,s,d),m(s,Ya,d),m(s,Rs,d),a(Rs,Ce),a(Rs,wa),a(wa,Ie),a(Rs,Ae),m(s,Na,d),T(Ds,s,d),m(s,Da,d),m(s,Gs,d),a(Gs,Ze),a(Gs,$a),a($a,Re),a(Gs,Ge),m(s,Qa,d),T(Qs,s,d),m(s,Sa,d),m(s,_s,d),a(_s,ze),a(_s,_a),a(_a,We),a(_s,Xe),a(_s,Ja),a(Ja,Be),a(_s,Fe),m(s,Ha,d),m(s,Js,d),a(Js,Ss),a(Ss,Ve),a(Ss,Hs),a(Hs,Ea),a(Ea,qe),a(Ss,Ye),a(Js,Ne),a(Js,bs),a(bs,De),a(bs,Ta),a(Ta,Qe),a(bs,Se),a(bs,ka),a(ka,He),a(bs,Pe),a(bs,va),a(va,Oe),a(bs,Le),a(Js,Ke),a(Js,Ps),a(Ps,st),a(Ps,Ua),a(Ua,at),a(Ps,et),m(s,Pa,d),m(s,pa,d),a(pa,tt),m(s,Oa,d),T(Os,s,d),m(s,La,d),m(s,Ms,d),a(Ms,lt),a(Ms,Ls),a(Ls,xa),a(xa,nt),a(Ms,ot),a(Ms,Ca),a(Ca,rt),a(Ms,pt),a(Ms,Ia),a(Ia,it),a(Ms,ct),m(s,Ka,d),T(Ks,s,d),m(s,se,d),m(s,rs,d),a(rs,mt),a(rs,Aa),a(Aa,dt),a(rs,ut),a(rs,Za),a(Za,ft),a(rs,ht),a(rs,Ra),a(Ra,jt),a(rs,Mt),a(rs,Ga),a(Ga,gt),a(rs,bt),m(s,ae,d),T(zs,s,d),m(s,ee,d),m(s,Us,d),a(Us,Ws),a(Ws,za),T(sa,za,null),a(Us,yt),a(Us,Wa),a(Wa,wt),m(s,te,d),T(Xs,s,d),m(s,le,d),T(Bs,s,d),ne=!0},p(s,[d]){const aa={};d&2&&(aa.$$scope={dirty:d,ctx:s}),N.$set(aa);const Xa={};d&2&&(Xa.$$scope={dirty:d,ctx:s}),zs.$set(Xa);const Ba={};d&2&&(Ba.$$scope={dirty:d,ctx:s}),Xs.$set(Ba);const Fa={};d&2&&(Fa.$$scope={dirty:d,ctx:s}),Bs.$set(Fa)},i(s){ne||(k(g.$$.fragment,s),k(Z.$$.fragment,s),k(N.$$.fragment,s),k(O.$$.fragment,s),k(D.$$.fragment,s),k(ss.$$.fragment,s),k(cs.$$.fragment,s),k(Ys.$$.fragment,s),k(Ns.$$.fragment,s),k(Ds.$$.fragment,s),k(Qs.$$.fragment,s),k(Os.$$.fragment,s),k(Ks.$$.fragment,s),k(zs.$$.fragment,s),k(sa.$$.fragment,s),k(Xs.$$.fragment,s),k(Bs.$$.fragment,s),ne=!0)},o(s){v(g.$$.fragment,s),v(Z.$$.fragment,s),v(N.$$.fragment,s),v(O.$$.fragment,s),v(D.$$.fragment,s),v(ss.$$.fragment,s),v(cs.$$.fragment,s),v(Ys.$$.fragment,s),v(Ns.$$.fragment,s),v(Ds.$$.fragment,s),v(Qs.$$.fragment,s),v(Os.$$.fragment,s),v(Ks.$$.fragment,s),v(zs.$$.fragment,s),v(sa.$$.fragment,s),v(Xs.$$.fragment,s),v(Bs.$$.fragment,s),ne=!1},d(s){e(t),s&&e(u),s&&e(l),U(g),s&&e(_),U(Z,s),s&&e(V),s&&e(P),s&&e(Q),s&&e(R),s&&e(hs),U(N,s),s&&e(js),s&&e(W),U(O),s&&e(es),s&&e(h),s&&e(ts),U(D,s),s&&e(K),s&&e(X),s&&e(H),U(ss,s),s&&e(ps),s&&e(ls),s&&e($),U(cs,s),s&&e(qs),s&&e(ms),s&&e(Ts),s&&e(ds),s&&e(Va),s&&e(vs),U(Ys),s&&e(qa),U(Ns,s),s&&e(Ya),s&&e(Rs),s&&e(Na),U(Ds,s),s&&e(Da),s&&e(Gs),s&&e(Qa),U(Qs,s),s&&e(Sa),s&&e(_s),s&&e(Ha),s&&e(Js),s&&e(Pa),s&&e(pa),s&&e(Oa),U(Os,s),s&&e(La),s&&e(Ms),s&&e(Ka),U(Ks,s),s&&e(se),s&&e(rs),s&&e(ae),U(zs,s),s&&e(ee),s&&e(Us),U(sa),s&&e(te),U(Xs,s),s&&e(le),U(Bs,s)}}}const Il={local:"classificao-de-tokens",sections:[{local:"carregando-o-conjunto-de-dados-wnut-17",title:"Carregando o conjunto de dados WNUT 17"},{local:"prprocessamento",title:"Pr\xE9-processamento"},{local:"treinamento",title:"Treinamento"}],title:"Classifica\xE7\xE3o de tokens"};function Al(A){return gl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Bl extends fl{constructor(t){super();hl(this,t,Al,Cl,jl,{})}}export{Bl as default,Il as metadata};
