import{S as os,i as ns,s as is,e as r,k as c,w as $,t as p,M as ps,c as o,d as e,m,a as n,x as M,h,b as d,G as t,g as f,y as j,q as k,o as T,B as _,v as hs}from"../chunks/vendor-hf-doc-builder.js";import{T as Fa}from"../chunks/Tip-hf-doc-builder.js";import{Y as Ba}from"../chunks/Youtube-hf-doc-builder.js";import{I as ea}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as Q}from"../chunks/CodeBlock-hf-doc-builder.js";import{D as fs}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as rs,M as za}from"../chunks/Markdown-hf-doc-builder.js";function ds(yt){let y,E;return{c(){y=r("p"),E=p(`You will see a warning about some of the pretrained weights not being used and some weights being randomly
initialized. Don\u2019t worry, this is completely normal! The pretrained head of the BERT model is discarded, and replaced with a randomly initialized classification head. You will fine-tune this new model head on your sequence classification task, transferring the knowledge of the pretrained model to it.`)},l(u){y=o(u,"P",{});var v=n(y);E=h(v,`You will see a warning about some of the pretrained weights not being used and some weights being randomly
initialized. Don\u2019t worry, this is completely normal! The pretrained head of the BERT model is discarded, and replaced with a randomly initialized classification head. You will fine-tune this new model head on your sequence classification task, transferring the knowledge of the pretrained model to it.`),v.forEach(e)},m(u,v){f(u,y,v),t(y,E)},d(u){u&&e(y)}}}function cs(yt){let y,E,u,v,J,Z,z,I,He,V,G,Qe,Y,ta,wt,R,we,jt,C,ge,qe,gt,rt,kt,Qt,Ce,ie,Le,q,B,ae,pe,he,ot,Be,Xt,Ct,N,nt,se,Tt,Ze,Fe,be,Bt,ve,it,bt,_t,L,De,P,x,Ue,F,pt,ht,le,fe,U,de,qt,Ie,ze,vt,ce,ft,$e,Me,Ve,re,aa,Et,je,Ye,D,K,Jt,ke,Ft,A,Ke,X,me,Oe,Ge,zt,et,sa,Vt,O,Ne,We,dt,S,tt,Re,Te,xe,at,ct,Zt,H,Yt,W,oe,ue,Ae,_e,mt,Ut,Lt,ee,Ee,ne,st,$t,Nt,l,w,ye,Je,xt,It,la,Xe,Pe,i;return y=new Ba({props:{id:"nvBXf7s7vTI"}}),Z=new ea({}),Ce=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmJlcnQtYmFzZS1jYXNlZCUyMiUyQyUyMG51bV9sYWJlbHMlM0Q1KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, num_labels=<span class="hljs-number">5</span>)`}}),Le=new Fa({props:{$$slots:{default:[ds]},$$scope:{ctx:yt}}}),he=new ea({}),L=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKG91dHB1dF9kaXIlM0QlMjJ0ZXN0X3RyYWluZXIlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(output_dir=<span class="hljs-string">&quot;test_trainer&quot;</span>)`}}),F=new ea({}),A=new Q({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBaW1wb3J0JTIwZXZhbHVhdGUlMEElMEFtZXRyaWMlMjAlM0QlMjBldmFsdWF0ZS5sb2FkKCUyMmFjY3VyYWN5JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> evaluate

<span class="hljs-meta">&gt;&gt;&gt; </span>metric = evaluate.load(<span class="hljs-string">&quot;accuracy&quot;</span>)`}}),S=new Q({props:{code:"ZGVmJTIwY29tcHV0ZV9tZXRyaWNzKGV2YWxfcHJlZCklM0ElMEElMjAlMjAlMjAlMjBsb2dpdHMlMkMlMjBsYWJlbHMlMjAlM0QlMjBldmFsX3ByZWQlMEElMjAlMjAlMjAlMjBwcmVkaWN0aW9ucyUyMCUzRCUyMG5wLmFyZ21heChsb2dpdHMlMkMlMjBheGlzJTNELTEpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwbWV0cmljLmNvbXB1dGUocHJlZGljdGlvbnMlM0RwcmVkaWN0aW9ucyUyQyUyMHJlZmVyZW5jZXMlM0RsYWJlbHMp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_pred</span>):
<span class="hljs-meta">... </span>    logits, labels = eval_pred
<span class="hljs-meta">... </span>    predictions = np.argmax(logits, axis=-<span class="hljs-number">1</span>)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> metric.compute(predictions=predictions, references=labels)`}}),H=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTJDJTIwVHJhaW5lciUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyhvdXRwdXRfZGlyJTNEJTIydGVzdF90cmFpbmVyJTIyJTJDJTIwZXZhbHVhdGlvbl9zdHJhdGVneSUzRCUyMmVwb2NoJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(output_dir=<span class="hljs-string">&quot;test_trainer&quot;</span>, evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>)`}}),Ae=new ea({}),l=new Q({props:{code:"dHJhaW5lciUyMCUzRCUyMFRyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRHNtYWxsX3RyYWluX2RhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBldmFsX2RhdGFzZXQlM0RzbWFsbF9ldmFsX2RhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBjb21wdXRlX21ldHJpY3MlM0Rjb21wdXRlX21ldHJpY3MlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=small_train_dataset,
<span class="hljs-meta">... </span>    eval_dataset=small_eval_dataset,
<span class="hljs-meta">... </span>    compute_metrics=compute_metrics,
<span class="hljs-meta">... </span>)`}}),Pe=new Q({props:{code:"dHJhaW5lci50cmFpbigp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()'}}),{c(){$(y.$$.fragment),E=c(),u=r("h2"),v=r("a"),J=r("span"),$(Z.$$.fragment),z=c(),I=r("span"),He=p("Train with PyTorch Trainer"),V=c(),G=r("p"),Qe=p("\u{1F917} Transformers provides a "),Y=r("a"),ta=p("Trainer"),wt=p(" class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),R=r("a"),we=p("Trainer"),jt=p(" API supports a wide range of training options and features such as logging, gradient accumulation, and mixed precision."),C=c(),ge=r("p"),qe=p("Start by loading your model and specify the number of expected labels. From the Yelp Review "),gt=r("a"),rt=p("dataset card"),kt=p(", you know there are five labels:"),Qt=c(),$(Ce.$$.fragment),ie=c(),$(Le.$$.fragment),q=c(),B=r("h3"),ae=r("a"),pe=r("span"),$(he.$$.fragment),ot=c(),Be=r("span"),Xt=p("Training hyperparameters"),Ct=c(),N=r("p"),nt=p("Next, create a "),se=r("a"),Tt=p("TrainingArguments"),Ze=p(" class which contains all the hyperparameters you can tune as well as flags for activating different training options. For this tutorial you can start with the default training "),Fe=r("a"),be=p("hyperparameters"),Bt=p(", but feel free to experiment with these to find your optimal settings."),ve=c(),it=r("p"),bt=p("Specify where to save the checkpoints from your training:"),_t=c(),$(L.$$.fragment),De=c(),P=r("h3"),x=r("a"),Ue=r("span"),$(F.$$.fragment),pt=c(),ht=r("span"),le=p("Evaluate"),fe=c(),U=r("p"),de=r("a"),qt=p("Trainer"),Ie=p(" does not automatically evaluate model performance during training. You\u2019ll need to pass "),ze=r("a"),vt=p("Trainer"),ce=p(" a function to compute and report metrics. The "),ft=r("a"),$e=p("\u{1F917} Evaluate"),Me=p(" library provides a simple "),Ve=r("a"),re=r("code"),aa=p("accuracy"),Et=p(" function you can load with the "),je=r("a"),Ye=p("evaluate.load"),D=p(" (see this "),K=r("a"),Jt=p("quicktour"),ke=p(" for more information) function:"),Ft=c(),$(A.$$.fragment),Ke=c(),X=r("p"),me=p("Call "),Oe=r("code"),Ge=p("compute"),zt=p(" on "),et=r("code"),sa=p("metric"),Vt=p(" to calculate the accuracy of your predictions. Before passing your predictions to "),O=r("code"),Ne=p("compute"),We=p(", you need to convert the predictions to logits (remember all \u{1F917} Transformers models return logits):"),dt=c(),$(S.$$.fragment),tt=c(),Re=r("p"),Te=p("If you\u2019d like to monitor your evaluation metrics during fine-tuning, specify the "),xe=r("code"),at=p("evaluation_strategy"),ct=p(" parameter in your training arguments to report the evaluation metric at the end of each epoch:"),Zt=c(),$(H.$$.fragment),Yt=c(),W=r("h3"),oe=r("a"),ue=r("span"),$(Ae.$$.fragment),_e=c(),mt=r("span"),Ut=p("Trainer"),Lt=c(),ee=r("p"),Ee=p("Create a "),ne=r("a"),st=p("Trainer"),$t=p(" object with your model, training arguments, training and test datasets, and evaluation function:"),Nt=c(),$(l.$$.fragment),w=c(),ye=r("p"),Je=p("Then fine-tune your model by calling "),xt=r("a"),It=p("train()"),la=p(":"),Xe=c(),$(Pe.$$.fragment),this.h()},l(a){M(y.$$.fragment,a),E=m(a),u=o(a,"H2",{class:!0});var b=n(u);v=o(b,"A",{id:!0,class:!0,href:!0});var Se=n(v);J=o(Se,"SPAN",{});var pa=n(J);M(Z.$$.fragment,pa),pa.forEach(e),Se.forEach(e),z=m(b),I=o(b,"SPAN",{});var Dt=n(I);He=h(Dt,"Train with PyTorch Trainer"),Dt.forEach(e),b.forEach(e),V=m(a),G=o(a,"P",{});var Gt=n(G);Qe=h(Gt,"\u{1F917} Transformers provides a "),Y=o(Gt,"A",{href:!0});var ha=n(Y);ta=h(ha,"Trainer"),ha.forEach(e),wt=h(Gt," class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),R=o(Gt,"A",{href:!0});var fa=n(R);we=h(fa,"Trainer"),fa.forEach(e),jt=h(Gt," API supports a wide range of training options and features such as logging, gradient accumulation, and mixed precision."),Gt.forEach(e),C=m(a),ge=o(a,"P",{});var ra=n(ge);qe=h(ra,"Start by loading your model and specify the number of expected labels. From the Yelp Review "),gt=o(ra,"A",{href:!0,rel:!0});var lt=n(gt);rt=h(lt,"dataset card"),lt.forEach(e),kt=h(ra,", you know there are five labels:"),ra.forEach(e),Qt=m(a),M(Ce.$$.fragment,a),ie=m(a),M(Le.$$.fragment,a),q=m(a),B=o(a,"H3",{class:!0});var Kt=n(B);ae=o(Kt,"A",{id:!0,class:!0,href:!0});var oa=n(ae);pe=o(oa,"SPAN",{});var ya=n(pe);M(he.$$.fragment,ya),ya.forEach(e),oa.forEach(e),ot=m(Kt),Be=o(Kt,"SPAN",{});var Wt=n(Be);Xt=h(Wt,"Training hyperparameters"),Wt.forEach(e),Kt.forEach(e),Ct=m(a),N=o(a,"P",{});var Rt=n(N);nt=h(Rt,"Next, create a "),se=o(Rt,"A",{href:!0});var wa=n(se);Tt=h(wa,"TrainingArguments"),wa.forEach(e),Ze=h(Rt," class which contains all the hyperparameters you can tune as well as flags for activating different training options. For this tutorial you can start with the default training "),Fe=o(Rt,"A",{href:!0,rel:!0});var ga=n(Fe);be=h(ga,"hyperparameters"),ga.forEach(e),Bt=h(Rt,", but feel free to experiment with these to find your optimal settings."),Rt.forEach(e),ve=m(a),it=o(a,"P",{});var da=n(it);bt=h(da,"Specify where to save the checkpoints from your training:"),da.forEach(e),_t=m(a),M(L.$$.fragment,a),De=m(a),P=o(a,"H3",{class:!0});var ut=n(P);x=o(ut,"A",{id:!0,class:!0,href:!0});var na=n(x);Ue=o(na,"SPAN",{});var s=n(Ue);M(F.$$.fragment,s),s.forEach(e),na.forEach(e),pt=m(ut),ht=o(ut,"SPAN",{});var g=n(ht);le=h(g,"Evaluate"),g.forEach(e),ut.forEach(e),fe=m(a),U=o(a,"P",{});var te=n(U);de=o(te,"A",{href:!0});var ia=n(de);qt=h(ia,"Trainer"),ia.forEach(e),Ie=h(te," does not automatically evaluate model performance during training. You\u2019ll need to pass "),ze=o(te,"A",{href:!0});var ca=n(ze);vt=h(ca,"Trainer"),ca.forEach(e),ce=h(te," a function to compute and report metrics. The "),ft=o(te,"A",{href:!0,rel:!0});var $a=n(ft);$e=h($a,"\u{1F917} Evaluate"),$a.forEach(e),Me=h(te," library provides a simple "),Ve=o(te,"A",{href:!0,rel:!0});var Pt=n(Ve);re=o(Pt,"CODE",{});var Ot=n(re);aa=h(Ot,"accuracy"),Ot.forEach(e),Pt.forEach(e),Et=h(te," function you can load with the "),je=o(te,"A",{href:!0,rel:!0});var ma=n(je);Ye=h(ma,"evaluate.load"),ma.forEach(e),D=h(te," (see this "),K=o(te,"A",{href:!0,rel:!0});var St=n(K);Jt=h(St,"quicktour"),St.forEach(e),ke=h(te," for more information) function:"),te.forEach(e),Ft=m(a),M(A.$$.fragment,a),Ke=m(a),X=o(a,"P",{});var Mt=n(X);me=h(Mt,"Call "),Oe=o(Mt,"CODE",{});var Ht=n(Oe);Ge=h(Ht,"compute"),Ht.forEach(e),zt=h(Mt," on "),et=o(Mt,"CODE",{});var ba=n(et);sa=h(ba,"metric"),ba.forEach(e),Vt=h(Mt," to calculate the accuracy of your predictions. Before passing your predictions to "),O=o(Mt,"CODE",{});var ua=n(O);Ne=h(ua,"compute"),ua.forEach(e),We=h(Mt,", you need to convert the predictions to logits (remember all \u{1F917} Transformers models return logits):"),Mt.forEach(e),dt=m(a),M(S.$$.fragment,a),tt=m(a),Re=o(a,"P",{});var Ma=n(Re);Te=h(Ma,"If you\u2019d like to monitor your evaluation metrics during fine-tuning, specify the "),xe=o(Ma,"CODE",{});var ja=n(xe);at=h(ja,"evaluation_strategy"),ja.forEach(e),ct=h(Ma," parameter in your training arguments to report the evaluation metric at the end of each epoch:"),Ma.forEach(e),Zt=m(a),M(H.$$.fragment,a),Yt=m(a),W=o(a,"H3",{class:!0});var At=n(W);oe=o(At,"A",{id:!0,class:!0,href:!0});var Ea=n(oe);ue=o(Ea,"SPAN",{});var ka=n(ue);M(Ae.$$.fragment,ka),ka.forEach(e),Ea.forEach(e),_e=m(At),mt=o(At,"SPAN",{});var Ja=n(mt);Ut=h(Ja,"Trainer"),Ja.forEach(e),At.forEach(e),Lt=m(a),ee=o(a,"P",{});var Ta=n(ee);Ee=h(Ta,"Create a "),ne=o(Ta,"A",{href:!0});var _a=n(ne);st=h(_a,"Trainer"),_a.forEach(e),$t=h(Ta," object with your model, training arguments, training and test datasets, and evaluation function:"),Ta.forEach(e),Nt=m(a),M(l.$$.fragment,a),w=m(a),ye=o(a,"P",{});var va=n(ye);Je=h(va,"Then fine-tune your model by calling "),xt=o(va,"A",{href:!0});var Ua=n(xt);It=h(Ua,"train()"),Ua.forEach(e),la=h(va,":"),va.forEach(e),Xe=m(a),M(Pe.$$.fragment,a),this.h()},h(){d(v,"id","train-with-pytorch-trainer"),d(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(v,"href","#train-with-pytorch-trainer"),d(u,"class","relative group"),d(Y,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),d(R,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),d(gt,"href","https://huggingface.co/datasets/yelp_review_full#data-fields"),d(gt,"rel","nofollow"),d(ae,"id","training-hyperparameters"),d(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ae,"href","#training-hyperparameters"),d(B,"class","relative group"),d(se,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.TrainingArguments"),d(Fe,"href","https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments"),d(Fe,"rel","nofollow"),d(x,"id","evaluate"),d(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(x,"href","#evaluate"),d(P,"class","relative group"),d(de,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),d(ze,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),d(ft,"href","https://huggingface.co/docs/evaluate/index"),d(ft,"rel","nofollow"),d(Ve,"href","https://huggingface.co/spaces/evaluate-metric/accuracy"),d(Ve,"rel","nofollow"),d(je,"href","https://huggingface.co/docs/evaluate/v0.4.0/en/package_reference/loading_methods#evaluate.load"),d(je,"rel","nofollow"),d(K,"href","https://huggingface.co/docs/evaluate/a_quick_tour"),d(K,"rel","nofollow"),d(oe,"id","trainer"),d(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(oe,"href","#trainer"),d(W,"class","relative group"),d(ne,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),d(xt,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer.train")},m(a,b){j(y,a,b),f(a,E,b),f(a,u,b),t(u,v),t(v,J),j(Z,J,null),t(u,z),t(u,I),t(I,He),f(a,V,b),f(a,G,b),t(G,Qe),t(G,Y),t(Y,ta),t(G,wt),t(G,R),t(R,we),t(G,jt),f(a,C,b),f(a,ge,b),t(ge,qe),t(ge,gt),t(gt,rt),t(ge,kt),f(a,Qt,b),j(Ce,a,b),f(a,ie,b),j(Le,a,b),f(a,q,b),f(a,B,b),t(B,ae),t(ae,pe),j(he,pe,null),t(B,ot),t(B,Be),t(Be,Xt),f(a,Ct,b),f(a,N,b),t(N,nt),t(N,se),t(se,Tt),t(N,Ze),t(N,Fe),t(Fe,be),t(N,Bt),f(a,ve,b),f(a,it,b),t(it,bt),f(a,_t,b),j(L,a,b),f(a,De,b),f(a,P,b),t(P,x),t(x,Ue),j(F,Ue,null),t(P,pt),t(P,ht),t(ht,le),f(a,fe,b),f(a,U,b),t(U,de),t(de,qt),t(U,Ie),t(U,ze),t(ze,vt),t(U,ce),t(U,ft),t(ft,$e),t(U,Me),t(U,Ve),t(Ve,re),t(re,aa),t(U,Et),t(U,je),t(je,Ye),t(U,D),t(U,K),t(K,Jt),t(U,ke),f(a,Ft,b),j(A,a,b),f(a,Ke,b),f(a,X,b),t(X,me),t(X,Oe),t(Oe,Ge),t(X,zt),t(X,et),t(et,sa),t(X,Vt),t(X,O),t(O,Ne),t(X,We),f(a,dt,b),j(S,a,b),f(a,tt,b),f(a,Re,b),t(Re,Te),t(Re,xe),t(xe,at),t(Re,ct),f(a,Zt,b),j(H,a,b),f(a,Yt,b),f(a,W,b),t(W,oe),t(oe,ue),j(Ae,ue,null),t(W,_e),t(W,mt),t(mt,Ut),f(a,Lt,b),f(a,ee,b),t(ee,Ee),t(ee,ne),t(ne,st),t(ee,$t),f(a,Nt,b),j(l,a,b),f(a,w,b),f(a,ye,b),t(ye,Je),t(ye,xt),t(xt,It),t(ye,la),f(a,Xe,b),j(Pe,a,b),i=!0},p(a,b){const Se={};b&2&&(Se.$$scope={dirty:b,ctx:a}),Le.$set(Se)},i(a){i||(k(y.$$.fragment,a),k(Z.$$.fragment,a),k(Ce.$$.fragment,a),k(Le.$$.fragment,a),k(he.$$.fragment,a),k(L.$$.fragment,a),k(F.$$.fragment,a),k(A.$$.fragment,a),k(S.$$.fragment,a),k(H.$$.fragment,a),k(Ae.$$.fragment,a),k(l.$$.fragment,a),k(Pe.$$.fragment,a),i=!0)},o(a){T(y.$$.fragment,a),T(Z.$$.fragment,a),T(Ce.$$.fragment,a),T(Le.$$.fragment,a),T(he.$$.fragment,a),T(L.$$.fragment,a),T(F.$$.fragment,a),T(A.$$.fragment,a),T(S.$$.fragment,a),T(H.$$.fragment,a),T(Ae.$$.fragment,a),T(l.$$.fragment,a),T(Pe.$$.fragment,a),i=!1},d(a){_(y,a),a&&e(E),a&&e(u),_(Z),a&&e(V),a&&e(G),a&&e(C),a&&e(ge),a&&e(Qt),_(Ce,a),a&&e(ie),_(Le,a),a&&e(q),a&&e(B),_(he),a&&e(Ct),a&&e(N),a&&e(ve),a&&e(it),a&&e(_t),_(L,a),a&&e(De),a&&e(P),_(F),a&&e(fe),a&&e(U),a&&e(Ft),_(A,a),a&&e(Ke),a&&e(X),a&&e(dt),_(S,a),a&&e(tt),a&&e(Re),a&&e(Zt),_(H,a),a&&e(Yt),a&&e(W),_(Ae),a&&e(Lt),a&&e(ee),a&&e(Nt),_(l,a),a&&e(w),a&&e(ye),a&&e(Xe),_(Pe,a)}}}function ms(yt){let y,E;return y=new za({props:{$$slots:{default:[cs]},$$scope:{ctx:yt}}}),{c(){$(y.$$.fragment)},l(u){M(y.$$.fragment,u)},m(u,v){j(y,u,v),E=!0},p(u,v){const J={};v&2&&(J.$$scope={dirty:v,ctx:u}),y.$set(J)},i(u){E||(k(y.$$.fragment,u),E=!0)},o(u){T(y.$$.fragment,u),E=!1},d(u){_(y,u)}}}function us(yt){let y,E,u,v,J;return{c(){y=r("p"),E=p("You don\u2019t have to pass a loss argument to your models when you "),u=r("code"),v=p("compile()"),J=p(` them! Hugging Face models automatically
choose a loss that is appropriate for their task and model architecture if this argument is left blank. You can always
override this by specifying a loss yourself if you want to!`)},l(Z){y=o(Z,"P",{});var z=n(y);E=h(z,"You don\u2019t have to pass a loss argument to your models when you "),u=o(z,"CODE",{});var I=n(u);v=h(I,"compile()"),I.forEach(e),J=h(z,` them! Hugging Face models automatically
choose a loss that is appropriate for their task and model architecture if this argument is left blank. You can always
override this by specifying a loss yourself if you want to!`),z.forEach(e)},m(Z,z){f(Z,y,z),t(y,E),t(y,u),t(u,v),t(y,J)},d(Z){Z&&e(y)}}}function ys(yt){let y,E,u,v,J,Z,z,I,He,V,G,Qe,Y,ta,wt,R,we,jt,C,ge,qe,gt,rt,kt,Qt,Ce,ie,Le,q,B,ae,pe,he,ot,Be,Xt,Ct,N,nt,se,Tt,Ze,Fe,be,Bt,ve,it,bt,_t,L,De,P,x,Ue,F,pt,ht,le,fe,U,de,qt,Ie,ze,vt,ce,ft,$e,Me,Ve,re,aa,Et,je,Ye,D,K,Jt,ke,Ft,A,Ke,X,me,Oe,Ge,zt,et,sa,Vt,O,Ne,We,dt,S,tt,Re,Te,xe,at,ct,Zt,H,Yt,W,oe,ue,Ae,_e,mt,Ut,Lt,ee,Ee,ne,st,$t,Nt,l,w,ye,Je,xt,It,la,Xe,Pe;return u=new Ba({props:{id:"rnTGBy2ax1c"}}),I=new ea({}),C=new ea({}),he=new Q({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJnbHVlJTIyJTJDJTIwJTIyY29sYSUyMiklMEFkYXRhc2V0JTIwJTNEJTIwZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTIwJTIwJTIzJTIwSnVzdCUyMHRha2UlMjB0aGUlMjB0cmFpbmluZyUyMHNwbGl0JTIwZm9yJTIwbm93",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

dataset = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;cola&quot;</span>)
dataset = dataset[<span class="hljs-string">&quot;train&quot;</span>]  <span class="hljs-comment"># Just take the training split for now</span>`}}),N=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIpJTBBdG9rZW5pemVkX2RhdGElMjAlM0QlMjB0b2tlbml6ZXIoZGF0YXNldCU1QiUyMnNlbnRlbmNlJTIyJTVEJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJucCUyMiUyQyUyMHBhZGRpbmclM0RUcnVlKSUwQSUyMyUyMFRva2VuaXplciUyMHJldHVybnMlMjBhJTIwQmF0Y2hFbmNvZGluZyUyQyUyMGJ1dCUyMHdlJTIwY29udmVydCUyMHRoYXQlMjB0byUyMGElMjBkaWN0JTIwZm9yJTIwS2VyYXMlMEF0b2tlbml6ZWRfZGF0YSUyMCUzRCUyMGRpY3QodG9rZW5pemVkX2RhdGEpJTBBJTBBbGFiZWxzJTIwJTNEJTIwbnAuYXJyYXkoZGF0YXNldCU1QiUyMmxhYmVsJTIyJTVEKSUyMCUyMCUyMyUyMExhYmVsJTIwaXMlMjBhbHJlYWR5JTIwYW4lMjBhcnJheSUyMG9mJTIwMCUyMGFuZCUyMDE=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
tokenized_data = tokenizer(dataset[<span class="hljs-string">&quot;sentence&quot;</span>], return_tensors=<span class="hljs-string">&quot;np&quot;</span>, padding=<span class="hljs-literal">True</span>)
<span class="hljs-comment"># Tokenizer returns a BatchEncoding, but we convert that to a dict for Keras</span>
tokenized_data = <span class="hljs-built_in">dict</span>(tokenized_data)

labels = np.array(dataset[<span class="hljs-string">&quot;label&quot;</span>])  <span class="hljs-comment"># Label is already an array of 0 and 1</span>`}}),De=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbiUwQWZyb20lMjB0ZW5zb3JmbG93LmtlcmFzLm9wdGltaXplcnMlMjBpbXBvcnQlMjBBZGFtJTBBJTBBJTIzJTIwTG9hZCUyMGFuZCUyMGNvbXBpbGUlMjBvdXIlMjBtb2RlbCUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JTZXF1ZW5jZUNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIpJTBBJTIzJTIwTG93ZXIlMjBsZWFybmluZyUyMHJhdGVzJTIwYXJlJTIwb2Z0ZW4lMjBiZXR0ZXIlMjBmb3IlMjBmaW5lLXR1bmluZyUyMHRyYW5zZm9ybWVycyUwQW1vZGVsLmNvbXBpbGUob3B0aW1pemVyJTNEQWRhbSgzZS01KSklMjAlMjAlMjMlMjBObyUyMGxvc3MlMjBhcmd1bWVudCElMEElMEFtb2RlbC5maXQodG9rZW5pemVkX2RhdGElMkMlMjBsYWJlbHMp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSequenceClassification
<span class="hljs-keyword">from</span> tensorflow.keras.optimizers <span class="hljs-keyword">import</span> Adam

<span class="hljs-comment"># Load and compile our model</span>
model = TFAutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
<span class="hljs-comment"># Lower learning rates are often better for fine-tuning transformers</span>
model.<span class="hljs-built_in">compile</span>(optimizer=Adam(<span class="hljs-number">3e-5</span>))  <span class="hljs-comment"># No loss argument!</span>

model.fit(tokenized_data, labels)`}}),x=new Fa({props:{$$slots:{default:[us]},$$scope:{ctx:yt}}}),de=new ea({}),Te=new Q({props:{code:"ZGVmJTIwdG9rZW5pemVfZGF0YXNldChkYXRhKSUzQSUwQSUyMCUyMCUyMCUyMCUyMyUyMEtleXMlMjBvZiUyMHRoZSUyMHJldHVybmVkJTIwZGljdGlvbmFyeSUyMHdpbGwlMjBiZSUyMGFkZGVkJTIwdG8lMjB0aGUlMjBkYXRhc2V0JTIwYXMlMjBjb2x1bW5zJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwdG9rZW5pemVyKGRhdGElNUIlMjJ0ZXh0JTIyJTVEKSUwQSUwQSUwQWRhdGFzZXQlMjAlM0QlMjBkYXRhc2V0Lm1hcCh0b2tlbml6ZV9kYXRhc2V0KQ==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_dataset</span>(<span class="hljs-params">data</span>):
    <span class="hljs-comment"># Keys of the returned dictionary will be added to the dataset as columns</span>
    <span class="hljs-keyword">return</span> tokenizer(data[<span class="hljs-string">&quot;text&quot;</span>])


dataset = dataset.<span class="hljs-built_in">map</span>(tokenize_dataset)`}}),H=new Q({props:{code:"dGZfZGF0YXNldCUyMCUzRCUyMG1vZGVsLnByZXBhcmVfdGZfZGF0YXNldChkYXRhc2V0JTVCJTIydHJhaW4lMjIlNUQlMkMlMjBiYXRjaF9zaXplJTNEMTYlMkMlMjBzaHVmZmxlJTNEVHJ1ZSUyQyUyMHRva2VuaXplciUzRHRva2VuaXplcik=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tf_dataset = model.prepare_tf_dataset(dataset[<span class="hljs-string">&quot;train&quot;</span>], batch_size=<span class="hljs-number">16</span>, shuffle=<span class="hljs-literal">True</span>, tokenizer=tokenizer)'}}),Xe=new Q({props:{code:"bW9kZWwuY29tcGlsZShvcHRpbWl6ZXIlM0RBZGFtKDNlLTUpKSUyMCUyMCUyMyUyME5vJTIwbG9zcyUyMGFyZ3VtZW50ISUwQSUwQW1vZGVsLmZpdCh0Zl9kYXRhc2V0KQ==",highlighted:`model.<span class="hljs-built_in">compile</span>(optimizer=Adam(<span class="hljs-number">3e-5</span>))  <span class="hljs-comment"># No loss argument!</span>

model.fit(tf_dataset)`}}),{c(){y=r("a"),E=c(),$(u.$$.fragment),v=c(),J=r("h2"),Z=r("a"),z=r("span"),$(I.$$.fragment),He=c(),V=r("span"),G=p("Train a TensorFlow model with Keras"),Qe=c(),Y=r("p"),ta=p("You can also train \u{1F917} Transformers models in TensorFlow with the Keras API!"),wt=c(),R=r("h3"),we=r("a"),jt=r("span"),$(C.$$.fragment),ge=c(),qe=r("span"),gt=p("Loading data for Keras"),rt=c(),kt=r("p"),Qt=p(`When you want to train a \u{1F917} Transformers model with the Keras API, you need to convert your dataset to a format that
Keras understands. If your dataset is small, you can just convert the whole thing to NumPy arrays and pass it to Keras.
Let\u2019s try that first before we do anything more complicated.`),Ce=c(),ie=r("p"),Le=p("First, load a dataset. We\u2019ll use the CoLA dataset from the "),q=r("a"),B=p("GLUE benchmark"),ae=p(`,
since it\u2019s a simple binary text classification task, and just take the training split for now.`),pe=c(),$(he.$$.fragment),ot=c(),Be=r("p"),Xt=p(`Next, load a tokenizer and tokenize the data as NumPy arrays. Note that the labels are already a list of 0 and 1s,
so we can just convert that directly to a NumPy array without tokenization!`),Ct=c(),$(N.$$.fragment),nt=c(),se=r("p"),Tt=p("Finally, load, "),Ze=r("a"),Fe=r("code"),be=p("compile"),Bt=p(", and "),ve=r("a"),it=r("code"),bt=p("fit"),_t=p(" the model. Note that Transformers models all have a default task-relevant loss function, so you don\u2019t need to specify one unless you want to:"),L=c(),$(De.$$.fragment),P=c(),$(x.$$.fragment),Ue=c(),F=r("p"),pt=p(`This approach works great for smaller datasets, but for larger datasets, you might find it starts to become a problem. Why?
Because the tokenized array and labels would have to be fully loaded into memory, and because NumPy doesn\u2019t handle
\u201Cjagged\u201D arrays, so every tokenized sample would have to be padded to the length of the longest sample in the whole
dataset. That\u2019s going to make your array even bigger, and all those padding tokens will slow down training too!`),ht=c(),le=r("h3"),fe=r("a"),U=r("span"),$(de.$$.fragment),qt=c(),Ie=r("span"),ze=p("Loading data as a tf.data.Dataset"),vt=c(),ce=r("p"),ft=p("If you want to avoid slowing down training, you can load your data as a "),$e=r("code"),Me=p("tf.data.Dataset"),Ve=p(` instead. Although you can write your own
`),re=r("code"),aa=p("tf.data"),Et=p(" pipeline if you want, we have two convenience methods for doing this:"),je=c(),Ye=r("ul"),D=r("li"),K=r("a"),Jt=p("prepare_tf_dataset()"),ke=p(`: This is the method we recommend in most cases. Because it is a method
on your model, it can inspect the model to automatically figure out which columns are usable as model inputs, and
discard the others to make a simpler, more performant dataset.`),Ft=c(),A=r("li"),Ke=r("a"),X=p("to_tf_dataset"),me=p(`: This method is more low-level, and is useful when you want to exactly control how
your dataset is created, by specifying exactly which `),Oe=r("code"),Ge=p("columns"),zt=p(" and "),et=r("code"),sa=p("label_cols"),Vt=p(" to include."),O=c(),Ne=r("p"),We=p("Before you can use "),dt=r("a"),S=p("prepare_tf_dataset()"),tt=p(`, you will need to add the tokenizer outputs to your dataset as columns, as shown in
the following code sample:`),Re=c(),$(Te.$$.fragment),xe=c(),at=r("p"),ct=p(`Remember that Hugging Face datasets are stored on disk by default, so this will not inflate your memory usage! Once the
columns have been added, you can stream batches from the dataset and add padding to each batch, which greatly
reduces the number of padding tokens compared to padding the entire dataset.`),Zt=c(),$(H.$$.fragment),Yt=c(),W=r("p"),oe=p("Note that in the code sample above, you need to pass the tokenizer to "),ue=r("code"),Ae=p("prepare_tf_dataset"),_e=p(` so it can correctly pad batches as they\u2019re loaded.
If all the samples in your dataset are the same length and no padding is necessary, you can skip this argument.
If you need to do something more complex than just padding samples (e.g. corrupting tokens for masked language
modelling), you can use the `),mt=r("code"),Ut=p("collate_fn"),Lt=p(` argument instead to pass a function that will be called to transform the
list of samples into a batch and apply any preprocessing you want. See our
`),ee=r("a"),Ee=p("examples"),ne=p(` or
`),st=r("a"),$t=p("notebooks"),Nt=p(" to see this approach in action."),l=c(),w=r("p"),ye=p("Once you\u2019ve created a "),Je=r("code"),xt=p("tf.data.Dataset"),It=p(", you can compile and fit the model as before:"),la=c(),$(Xe.$$.fragment),this.h()},l(i){y=o(i,"A",{id:!0}),n(y).forEach(e),E=m(i),M(u.$$.fragment,i),v=m(i),J=o(i,"H2",{class:!0});var a=n(J);Z=o(a,"A",{id:!0,class:!0,href:!0});var b=n(Z);z=o(b,"SPAN",{});var Se=n(z);M(I.$$.fragment,Se),Se.forEach(e),b.forEach(e),He=m(a),V=o(a,"SPAN",{});var pa=n(V);G=h(pa,"Train a TensorFlow model with Keras"),pa.forEach(e),a.forEach(e),Qe=m(i),Y=o(i,"P",{});var Dt=n(Y);ta=h(Dt,"You can also train \u{1F917} Transformers models in TensorFlow with the Keras API!"),Dt.forEach(e),wt=m(i),R=o(i,"H3",{class:!0});var Gt=n(R);we=o(Gt,"A",{id:!0,class:!0,href:!0});var ha=n(we);jt=o(ha,"SPAN",{});var fa=n(jt);M(C.$$.fragment,fa),fa.forEach(e),ha.forEach(e),ge=m(Gt),qe=o(Gt,"SPAN",{});var ra=n(qe);gt=h(ra,"Loading data for Keras"),ra.forEach(e),Gt.forEach(e),rt=m(i),kt=o(i,"P",{});var lt=n(kt);Qt=h(lt,`When you want to train a \u{1F917} Transformers model with the Keras API, you need to convert your dataset to a format that
Keras understands. If your dataset is small, you can just convert the whole thing to NumPy arrays and pass it to Keras.
Let\u2019s try that first before we do anything more complicated.`),lt.forEach(e),Ce=m(i),ie=o(i,"P",{});var Kt=n(ie);Le=h(Kt,"First, load a dataset. We\u2019ll use the CoLA dataset from the "),q=o(Kt,"A",{href:!0,rel:!0});var oa=n(q);B=h(oa,"GLUE benchmark"),oa.forEach(e),ae=h(Kt,`,
since it\u2019s a simple binary text classification task, and just take the training split for now.`),Kt.forEach(e),pe=m(i),M(he.$$.fragment,i),ot=m(i),Be=o(i,"P",{});var ya=n(Be);Xt=h(ya,`Next, load a tokenizer and tokenize the data as NumPy arrays. Note that the labels are already a list of 0 and 1s,
so we can just convert that directly to a NumPy array without tokenization!`),ya.forEach(e),Ct=m(i),M(N.$$.fragment,i),nt=m(i),se=o(i,"P",{});var Wt=n(se);Tt=h(Wt,"Finally, load, "),Ze=o(Wt,"A",{href:!0,rel:!0});var Rt=n(Ze);Fe=o(Rt,"CODE",{});var wa=n(Fe);be=h(wa,"compile"),wa.forEach(e),Rt.forEach(e),Bt=h(Wt,", and "),ve=o(Wt,"A",{href:!0,rel:!0});var ga=n(ve);it=o(ga,"CODE",{});var da=n(it);bt=h(da,"fit"),da.forEach(e),ga.forEach(e),_t=h(Wt," the model. Note that Transformers models all have a default task-relevant loss function, so you don\u2019t need to specify one unless you want to:"),Wt.forEach(e),L=m(i),M(De.$$.fragment,i),P=m(i),M(x.$$.fragment,i),Ue=m(i),F=o(i,"P",{});var ut=n(F);pt=h(ut,`This approach works great for smaller datasets, but for larger datasets, you might find it starts to become a problem. Why?
Because the tokenized array and labels would have to be fully loaded into memory, and because NumPy doesn\u2019t handle
\u201Cjagged\u201D arrays, so every tokenized sample would have to be padded to the length of the longest sample in the whole
dataset. That\u2019s going to make your array even bigger, and all those padding tokens will slow down training too!`),ut.forEach(e),ht=m(i),le=o(i,"H3",{class:!0});var na=n(le);fe=o(na,"A",{id:!0,class:!0,href:!0});var s=n(fe);U=o(s,"SPAN",{});var g=n(U);M(de.$$.fragment,g),g.forEach(e),s.forEach(e),qt=m(na),Ie=o(na,"SPAN",{});var te=n(Ie);ze=h(te,"Loading data as a tf.data.Dataset"),te.forEach(e),na.forEach(e),vt=m(i),ce=o(i,"P",{});var ia=n(ce);ft=h(ia,"If you want to avoid slowing down training, you can load your data as a "),$e=o(ia,"CODE",{});var ca=n($e);Me=h(ca,"tf.data.Dataset"),ca.forEach(e),Ve=h(ia,` instead. Although you can write your own
`),re=o(ia,"CODE",{});var $a=n(re);aa=h($a,"tf.data"),$a.forEach(e),Et=h(ia," pipeline if you want, we have two convenience methods for doing this:"),ia.forEach(e),je=m(i),Ye=o(i,"UL",{});var Pt=n(Ye);D=o(Pt,"LI",{});var Ot=n(D);K=o(Ot,"A",{href:!0});var ma=n(K);Jt=h(ma,"prepare_tf_dataset()"),ma.forEach(e),ke=h(Ot,`: This is the method we recommend in most cases. Because it is a method
on your model, it can inspect the model to automatically figure out which columns are usable as model inputs, and
discard the others to make a simpler, more performant dataset.`),Ot.forEach(e),Ft=m(Pt),A=o(Pt,"LI",{});var St=n(A);Ke=o(St,"A",{href:!0,rel:!0});var Mt=n(Ke);X=h(Mt,"to_tf_dataset"),Mt.forEach(e),me=h(St,`: This method is more low-level, and is useful when you want to exactly control how
your dataset is created, by specifying exactly which `),Oe=o(St,"CODE",{});var Ht=n(Oe);Ge=h(Ht,"columns"),Ht.forEach(e),zt=h(St," and "),et=o(St,"CODE",{});var ba=n(et);sa=h(ba,"label_cols"),ba.forEach(e),Vt=h(St," to include."),St.forEach(e),Pt.forEach(e),O=m(i),Ne=o(i,"P",{});var ua=n(Ne);We=h(ua,"Before you can use "),dt=o(ua,"A",{href:!0});var Ma=n(dt);S=h(Ma,"prepare_tf_dataset()"),Ma.forEach(e),tt=h(ua,`, you will need to add the tokenizer outputs to your dataset as columns, as shown in
the following code sample:`),ua.forEach(e),Re=m(i),M(Te.$$.fragment,i),xe=m(i),at=o(i,"P",{});var ja=n(at);ct=h(ja,`Remember that Hugging Face datasets are stored on disk by default, so this will not inflate your memory usage! Once the
columns have been added, you can stream batches from the dataset and add padding to each batch, which greatly
reduces the number of padding tokens compared to padding the entire dataset.`),ja.forEach(e),Zt=m(i),M(H.$$.fragment,i),Yt=m(i),W=o(i,"P",{});var At=n(W);oe=h(At,"Note that in the code sample above, you need to pass the tokenizer to "),ue=o(At,"CODE",{});var Ea=n(ue);Ae=h(Ea,"prepare_tf_dataset"),Ea.forEach(e),_e=h(At,` so it can correctly pad batches as they\u2019re loaded.
If all the samples in your dataset are the same length and no padding is necessary, you can skip this argument.
If you need to do something more complex than just padding samples (e.g. corrupting tokens for masked language
modelling), you can use the `),mt=o(At,"CODE",{});var ka=n(mt);Ut=h(ka,"collate_fn"),ka.forEach(e),Lt=h(At,` argument instead to pass a function that will be called to transform the
list of samples into a batch and apply any preprocessing you want. See our
`),ee=o(At,"A",{href:!0,rel:!0});var Ja=n(ee);Ee=h(Ja,"examples"),Ja.forEach(e),ne=h(At,` or
`),st=o(At,"A",{href:!0,rel:!0});var Ta=n(st);$t=h(Ta,"notebooks"),Ta.forEach(e),Nt=h(At," to see this approach in action."),At.forEach(e),l=m(i),w=o(i,"P",{});var _a=n(w);ye=h(_a,"Once you\u2019ve created a "),Je=o(_a,"CODE",{});var va=n(Je);xt=h(va,"tf.data.Dataset"),va.forEach(e),It=h(_a,", you can compile and fit the model as before:"),_a.forEach(e),la=m(i),M(Xe.$$.fragment,i),this.h()},h(){d(y,"id","keras"),d(Z,"id","train-a-tensorflow-model-with-keras"),d(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Z,"href","#train-a-tensorflow-model-with-keras"),d(J,"class","relative group"),d(we,"id","loading-data-for-keras"),d(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(we,"href","#loading-data-for-keras"),d(R,"class","relative group"),d(q,"href","https://huggingface.co/datasets/glue"),d(q,"rel","nofollow"),d(Ze,"href","https://keras.io/api/models/model_training_apis/#compile-method"),d(Ze,"rel","nofollow"),d(ve,"href","https://keras.io/api/models/model_training_apis/#fit-method"),d(ve,"rel","nofollow"),d(fe,"id","loading-data-as-a-tfdatadataset"),d(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(fe,"href","#loading-data-as-a-tfdatadataset"),d(le,"class","relative group"),d(K,"href","/docs/transformers/v4.31.0/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset"),d(Ke,"href","https://huggingface.co/docs/datasets/v2.13.1/en/package_reference/main_classes#datasets.Dataset.to_tf_dataset"),d(Ke,"rel","nofollow"),d(dt,"href","/docs/transformers/v4.31.0/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset"),d(ee,"href","https://github.com/huggingface/transformers/tree/main/examples"),d(ee,"rel","nofollow"),d(st,"href","https://huggingface.co/docs/transformers/notebooks"),d(st,"rel","nofollow")},m(i,a){f(i,y,a),f(i,E,a),j(u,i,a),f(i,v,a),f(i,J,a),t(J,Z),t(Z,z),j(I,z,null),t(J,He),t(J,V),t(V,G),f(i,Qe,a),f(i,Y,a),t(Y,ta),f(i,wt,a),f(i,R,a),t(R,we),t(we,jt),j(C,jt,null),t(R,ge),t(R,qe),t(qe,gt),f(i,rt,a),f(i,kt,a),t(kt,Qt),f(i,Ce,a),f(i,ie,a),t(ie,Le),t(ie,q),t(q,B),t(ie,ae),f(i,pe,a),j(he,i,a),f(i,ot,a),f(i,Be,a),t(Be,Xt),f(i,Ct,a),j(N,i,a),f(i,nt,a),f(i,se,a),t(se,Tt),t(se,Ze),t(Ze,Fe),t(Fe,be),t(se,Bt),t(se,ve),t(ve,it),t(it,bt),t(se,_t),f(i,L,a),j(De,i,a),f(i,P,a),j(x,i,a),f(i,Ue,a),f(i,F,a),t(F,pt),f(i,ht,a),f(i,le,a),t(le,fe),t(fe,U),j(de,U,null),t(le,qt),t(le,Ie),t(Ie,ze),f(i,vt,a),f(i,ce,a),t(ce,ft),t(ce,$e),t($e,Me),t(ce,Ve),t(ce,re),t(re,aa),t(ce,Et),f(i,je,a),f(i,Ye,a),t(Ye,D),t(D,K),t(K,Jt),t(D,ke),t(Ye,Ft),t(Ye,A),t(A,Ke),t(Ke,X),t(A,me),t(A,Oe),t(Oe,Ge),t(A,zt),t(A,et),t(et,sa),t(A,Vt),f(i,O,a),f(i,Ne,a),t(Ne,We),t(Ne,dt),t(dt,S),t(Ne,tt),f(i,Re,a),j(Te,i,a),f(i,xe,a),f(i,at,a),t(at,ct),f(i,Zt,a),j(H,i,a),f(i,Yt,a),f(i,W,a),t(W,oe),t(W,ue),t(ue,Ae),t(W,_e),t(W,mt),t(mt,Ut),t(W,Lt),t(W,ee),t(ee,Ee),t(W,ne),t(W,st),t(st,$t),t(W,Nt),f(i,l,a),f(i,w,a),t(w,ye),t(w,Je),t(Je,xt),t(w,It),f(i,la,a),j(Xe,i,a),Pe=!0},p(i,a){const b={};a&2&&(b.$$scope={dirty:a,ctx:i}),x.$set(b)},i(i){Pe||(k(u.$$.fragment,i),k(I.$$.fragment,i),k(C.$$.fragment,i),k(he.$$.fragment,i),k(N.$$.fragment,i),k(De.$$.fragment,i),k(x.$$.fragment,i),k(de.$$.fragment,i),k(Te.$$.fragment,i),k(H.$$.fragment,i),k(Xe.$$.fragment,i),Pe=!0)},o(i){T(u.$$.fragment,i),T(I.$$.fragment,i),T(C.$$.fragment,i),T(he.$$.fragment,i),T(N.$$.fragment,i),T(De.$$.fragment,i),T(x.$$.fragment,i),T(de.$$.fragment,i),T(Te.$$.fragment,i),T(H.$$.fragment,i),T(Xe.$$.fragment,i),Pe=!1},d(i){i&&e(y),i&&e(E),_(u,i),i&&e(v),i&&e(J),_(I),i&&e(Qe),i&&e(Y),i&&e(wt),i&&e(R),_(C),i&&e(rt),i&&e(kt),i&&e(Ce),i&&e(ie),i&&e(pe),_(he,i),i&&e(ot),i&&e(Be),i&&e(Ct),_(N,i),i&&e(nt),i&&e(se),i&&e(L),_(De,i),i&&e(P),_(x,i),i&&e(Ue),i&&e(F),i&&e(ht),i&&e(le),_(de),i&&e(vt),i&&e(ce),i&&e(je),i&&e(Ye),i&&e(O),i&&e(Ne),i&&e(Re),_(Te,i),i&&e(xe),i&&e(at),i&&e(Zt),_(H,i),i&&e(Yt),i&&e(W),i&&e(l),i&&e(w),i&&e(la),_(Xe,i)}}}function ws(yt){let y,E;return y=new za({props:{$$slots:{default:[ys]},$$scope:{ctx:yt}}}),{c(){$(y.$$.fragment)},l(u){M(y.$$.fragment,u)},m(u,v){j(y,u,v),E=!0},p(u,v){const J={};v&2&&(J.$$scope={dirty:v,ctx:u}),y.$set(J)},i(u){E||(k(y.$$.fragment,u),E=!0)},o(u){T(y.$$.fragment,u),E=!1},d(u){_(y,u)}}}function gs(yt){let y,E,u,v,J,Z,z,I;return{c(){y=r("p"),E=p("Get free access to a cloud GPU if you don\u2019t have one with a hosted notebook like "),u=r("a"),v=p("Colaboratory"),J=p(" or "),Z=r("a"),z=p("SageMaker StudioLab"),I=p("."),this.h()},l(He){y=o(He,"P",{});var V=n(y);E=h(V,"Get free access to a cloud GPU if you don\u2019t have one with a hosted notebook like "),u=o(V,"A",{href:!0,rel:!0});var G=n(u);v=h(G,"Colaboratory"),G.forEach(e),J=h(V," or "),Z=o(V,"A",{href:!0,rel:!0});var Qe=n(Z);z=h(Qe,"SageMaker StudioLab"),Qe.forEach(e),I=h(V,"."),V.forEach(e),this.h()},h(){d(u,"href","https://colab.research.google.com/"),d(u,"rel","nofollow"),d(Z,"href","https://studiolab.sagemaker.aws/"),d(Z,"rel","nofollow")},m(He,V){f(He,y,V),t(y,E),t(y,u),t(u,v),t(y,J),t(y,Z),t(Z,z),t(y,I)},d(He){He&&e(y)}}}function bs(yt){let y,E,u,v,J,Z,z,I,He,V,G,Qe,Y,ta,wt,R,we,jt,C,ge,qe,gt,rt,kt,Qt,Ce,ie,Le,q,B,ae,pe,he,ot,Be,Xt,Ct,N,nt,se,Tt,Ze,Fe,be,Bt,ve,it,bt,_t,L,De,P,x,Ue,F,pt,ht,le,fe,U,de,qt,Ie,ze,vt,ce,ft,$e,Me,Ve,re,aa,Et,je,Ye,D,K,Jt,ke,Ft,A,Ke,X,me,Oe,Ge,zt,et,sa,Vt,O,Ne,We,dt,S,tt,Re,Te,xe,at,ct,Zt,H,Yt,W,oe,ue,Ae,_e,mt,Ut,Lt,ee,Ee,ne,st,$t,Nt,l,w,ye,Je,xt,It,la,Xe,Pe,i,a,b,Se,pa,Dt,Gt,ha,fa,ra,lt,Kt,oa,ya,Wt,Rt,wa,ga,da,ut,na;return y=new Ba({props:{id:"Dh9CL8fyG80"}}),G=new Q({props:{code:"ZGVsJTIwbW9kZWwlMEFkZWwlMjB0cmFpbmVyJTBBdG9yY2guY3VkYS5lbXB0eV9jYWNoZSgp",highlighted:`<span class="hljs-keyword">del</span> model
<span class="hljs-keyword">del</span> trainer
torch.cuda.empty_cache()`}}),ie=new Q({props:{code:"dG9rZW5pemVkX2RhdGFzZXRzJTIwJTNEJTIwdG9rZW5pemVkX2RhdGFzZXRzLnJlbW92ZV9jb2x1bW5zKCU1QiUyMnRleHQlMjIlNUQp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets = tokenized_datasets.remove_columns([<span class="hljs-string">&quot;text&quot;</span>])'}}),Ze=new Q({props:{code:"dG9rZW5pemVkX2RhdGFzZXRzJTIwJTNEJTIwdG9rZW5pemVkX2RhdGFzZXRzLnJlbmFtZV9jb2x1bW4oJTIybGFiZWwlMjIlMkMlMjAlMjJsYWJlbHMlMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets = tokenized_datasets.rename_column(<span class="hljs-string">&quot;label&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>)'}}),bt=new Q({props:{code:"dG9rZW5pemVkX2RhdGFzZXRzLnNldF9mb3JtYXQoJTIydG9yY2glMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets.set_format(<span class="hljs-string">&quot;torch&quot;</span>)'}}),x=new Q({props:{code:"c21hbGxfdHJhaW5fZGF0YXNldCUyMCUzRCUyMHRva2VuaXplZF9kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVELnNodWZmbGUoc2VlZCUzRDQyKS5zZWxlY3QocmFuZ2UoMTAwMCkpJTBBc21hbGxfZXZhbF9kYXRhc2V0JTIwJTNEJTIwdG9rZW5pemVkX2RhdGFzZXRzJTVCJTIydGVzdCUyMiU1RC5zaHVmZmxlKHNlZWQlM0Q0Mikuc2VsZWN0KHJhbmdlKDEwMDApKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>small_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>small_eval_dataset = tokenized_datasets[<span class="hljs-string">&quot;test&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))`}}),le=new ea({}),Me=new Q({props:{code:"ZnJvbSUyMHRvcmNoLnV0aWxzLmRhdGElMjBpbXBvcnQlMjBEYXRhTG9hZGVyJTBBJTBBdHJhaW5fZGF0YWxvYWRlciUyMCUzRCUyMERhdGFMb2FkZXIoc21hbGxfdHJhaW5fZGF0YXNldCUyQyUyMHNodWZmbGUlM0RUcnVlJTJDJTIwYmF0Y2hfc2l6ZSUzRDgpJTBBZXZhbF9kYXRhbG9hZGVyJTIwJTNEJTIwRGF0YUxvYWRlcihzbWFsbF9ldmFsX2RhdGFzZXQlMkMlMjBiYXRjaF9zaXplJTNEOCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

<span class="hljs-meta">&gt;&gt;&gt; </span>train_dataloader = DataLoader(small_train_dataset, shuffle=<span class="hljs-literal">True</span>, batch_size=<span class="hljs-number">8</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>eval_dataloader = DataLoader(small_eval_dataset, batch_size=<span class="hljs-number">8</span>)`}}),je=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmJlcnQtYmFzZS1jYXNlZCUyMiUyQyUyMG51bV9sYWJlbHMlM0Q1KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, num_labels=<span class="hljs-number">5</span>)`}}),ke=new ea({}),O=new Q({props:{code:"ZnJvbSUyMHRvcmNoLm9wdGltJTIwaW1wb3J0JTIwQWRhbVclMEElMEFvcHRpbWl6ZXIlMjAlM0QlMjBBZGFtVyhtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMjBsciUzRDVlLTUp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamW(model.parameters(), lr=<span class="hljs-number">5e-5</span>)`}}),xe=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGdldF9zY2hlZHVsZXIlMEElMEFudW1fZXBvY2hzJTIwJTNEJTIwMyUwQW51bV90cmFpbmluZ19zdGVwcyUyMCUzRCUyMG51bV9lcG9jaHMlMjAqJTIwbGVuKHRyYWluX2RhdGFsb2FkZXIpJTBBbHJfc2NoZWR1bGVyJTIwJTNEJTIwZ2V0X3NjaGVkdWxlciglMEElMjAlMjAlMjAlMjBuYW1lJTNEJTIybGluZWFyJTIyJTJDJTIwb3B0aW1pemVyJTNEb3B0aW1pemVyJTJDJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMjBudW1fdHJhaW5pbmdfc3RlcHMlM0RudW1fdHJhaW5pbmdfc3RlcHMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

<span class="hljs-meta">&gt;&gt;&gt; </span>num_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_training_steps = num_epochs * <span class="hljs-built_in">len</span>(train_dataloader)
<span class="hljs-meta">&gt;&gt;&gt; </span>lr_scheduler = get_scheduler(
<span class="hljs-meta">... </span>    name=<span class="hljs-string">&quot;linear&quot;</span>, optimizer=optimizer, num_warmup_steps=<span class="hljs-number">0</span>, num_training_steps=num_training_steps
<span class="hljs-meta">... </span>)`}}),ue=new Q({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFkZXZpY2UlMjAlM0QlMjB0b3JjaC5kZXZpY2UoJTIyY3VkYSUyMiklMjBpZiUyMHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklMjBlbHNlJTIwdG9yY2guZGV2aWNlKCUyMmNwdSUyMiklMEFtb2RlbC50byhkZXZpY2Up",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>) <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.to(device)`}}),_e=new Fa({props:{$$slots:{default:[gs]},$$scope:{ctx:yt}}}),$t=new ea({}),i=new Q({props:{code:"ZnJvbSUyMHRxZG0uYXV0byUyMGltcG9ydCUyMHRxZG0lMEElMEFwcm9ncmVzc19iYXIlMjAlM0QlMjB0cWRtKHJhbmdlKG51bV90cmFpbmluZ19zdGVwcykpJTBBJTBBbW9kZWwudHJhaW4oKSUwQWZvciUyMGVwb2NoJTIwaW4lMjByYW5nZShudW1fZXBvY2hzKSUzQSUwQSUyMCUyMCUyMCUyMGZvciUyMGJhdGNoJTIwaW4lMjB0cmFpbl9kYXRhbG9hZGVyJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYmF0Y2glMjAlM0QlMjAlN0JrJTNBJTIwdi50byhkZXZpY2UpJTIwZm9yJTIwayUyQyUyMHYlMjBpbiUyMGJhdGNoLml0ZW1zKCklN0QlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwb3V0cHV0cy5sb3NzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbG9zcy5iYWNrd2FyZCgpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxyX3NjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcm9ncmVzc19iYXIudXBkYXRlKDEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm

<span class="hljs-meta">&gt;&gt;&gt; </span>progress_bar = tqdm(<span class="hljs-built_in">range</span>(num_training_steps))

<span class="hljs-meta">&gt;&gt;&gt; </span>model.train()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_epochs):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
<span class="hljs-meta">... </span>        batch = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()}
<span class="hljs-meta">... </span>        outputs = model(**batch)
<span class="hljs-meta">... </span>        loss = outputs.loss
<span class="hljs-meta">... </span>        loss.backward()

<span class="hljs-meta">... </span>        optimizer.step()
<span class="hljs-meta">... </span>        lr_scheduler.step()
<span class="hljs-meta">... </span>        optimizer.zero_grad()
<span class="hljs-meta">... </span>        progress_bar.update(<span class="hljs-number">1</span>)`}}),Dt=new ea({}),ut=new Q({props:{code:"aW1wb3J0JTIwZXZhbHVhdGUlMEElMEFtZXRyaWMlMjAlM0QlMjBldmFsdWF0ZS5sb2FkKCUyMmFjY3VyYWN5JTIyKSUwQW1vZGVsLmV2YWwoKSUwQWZvciUyMGJhdGNoJTIwaW4lMjBldmFsX2RhdGFsb2FkZXIlM0ElMEElMjAlMjAlMjAlMjBiYXRjaCUyMCUzRCUyMCU3QmslM0ElMjB2LnRvKGRldmljZSklMjBmb3IlMjBrJTJDJTIwdiUyMGluJTIwYmF0Y2guaXRlbXMoKSU3RCUwQSUyMCUyMCUyMCUyMHdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKipiYXRjaCklMEElMEElMjAlMjAlMjAlMjBsb2dpdHMlMjAlM0QlMjBvdXRwdXRzLmxvZ2l0cyUwQSUyMCUyMCUyMCUyMHByZWRpY3Rpb25zJTIwJTNEJTIwdG9yY2guYXJnbWF4KGxvZ2l0cyUyQyUyMGRpbSUzRC0xKSUwQSUyMCUyMCUyMCUyMG1ldHJpYy5hZGRfYmF0Y2gocHJlZGljdGlvbnMlM0RwcmVkaWN0aW9ucyUyQyUyMHJlZmVyZW5jZXMlM0RiYXRjaCU1QiUyMmxhYmVscyUyMiU1RCklMEElMEFtZXRyaWMuY29tcHV0ZSgp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> evaluate

<span class="hljs-meta">&gt;&gt;&gt; </span>metric = evaluate.load(<span class="hljs-string">&quot;accuracy&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">eval</span>()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> eval_dataloader:
<span class="hljs-meta">... </span>    batch = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()}
<span class="hljs-meta">... </span>    <span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>        outputs = model(**batch)

<span class="hljs-meta">... </span>    logits = outputs.logits
<span class="hljs-meta">... </span>    predictions = torch.argmax(logits, dim=-<span class="hljs-number">1</span>)
<span class="hljs-meta">... </span>    metric.add_batch(predictions=predictions, references=batch[<span class="hljs-string">&quot;labels&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>metric.compute()`}}),{c(){$(y.$$.fragment),E=c(),u=r("p"),v=r("a"),J=p("Trainer"),Z=p(" takes care of the training loop and allows you to fine-tune a model in a single line of code. For users who prefer to write their own training loop, you can also fine-tune a \u{1F917} Transformers model in native PyTorch."),z=c(),I=r("p"),He=p("At this point, you may need to restart your notebook or execute the following code to free some memory:"),V=c(),$(G.$$.fragment),Qe=c(),Y=r("p"),ta=p("Next, manually postprocess "),wt=r("code"),R=p("tokenized_dataset"),we=p(" to prepare it for training."),jt=c(),C=r("ol"),ge=r("li"),qe=r("p"),gt=p("Remove the "),rt=r("code"),kt=p("text"),Qt=p(" column because the model does not accept raw text as an input:"),Ce=c(),$(ie.$$.fragment),Le=c(),q=r("li"),B=r("p"),ae=p("Rename the "),pe=r("code"),he=p("label"),ot=p(" column to "),Be=r("code"),Xt=p("labels"),Ct=p(" because the model expects the argument to be named "),N=r("code"),nt=p("labels"),se=p(":"),Tt=c(),$(Ze.$$.fragment),Fe=c(),be=r("li"),Bt=r("p"),ve=p("Set the format of the dataset to return PyTorch tensors instead of lists:"),it=c(),$(bt.$$.fragment),_t=c(),L=r("p"),De=p("Then create a smaller subset of the dataset as previously shown to speed up the fine-tuning:"),P=c(),$(x.$$.fragment),Ue=c(),F=r("h3"),pt=r("a"),ht=r("span"),$(le.$$.fragment),fe=c(),U=r("span"),de=p("DataLoader"),qt=c(),Ie=r("p"),ze=p("Create a "),vt=r("code"),ce=p("DataLoader"),ft=p(" for your training and test datasets so you can iterate over batches of data:"),$e=c(),$(Me.$$.fragment),Ve=c(),re=r("p"),aa=p("Load your model with the number of expected labels:"),Et=c(),$(je.$$.fragment),Ye=c(),D=r("h3"),K=r("a"),Jt=r("span"),$(ke.$$.fragment),Ft=c(),A=r("span"),Ke=p("Optimizer and learning rate scheduler"),X=c(),me=r("p"),Oe=p("Create an optimizer and learning rate scheduler to fine-tune the model. Let\u2019s use the "),Ge=r("a"),zt=r("code"),et=p("AdamW"),sa=p(" optimizer from PyTorch:"),Vt=c(),$(O.$$.fragment),Ne=c(),We=r("p"),dt=p("Create the default learning rate scheduler from "),S=r("a"),tt=p("Trainer"),Re=p(":"),Te=c(),$(xe.$$.fragment),at=c(),ct=r("p"),Zt=p("Lastly, specify "),H=r("code"),Yt=p("device"),W=p(" to use a GPU if you have access to one. Otherwise, training on a CPU may take several hours instead of a couple of minutes."),oe=c(),$(ue.$$.fragment),Ae=c(),$(_e.$$.fragment),mt=c(),Ut=r("p"),Lt=p("Great, now you are ready to train! \u{1F973}"),ee=c(),Ee=r("h3"),ne=r("a"),st=r("span"),$($t.$$.fragment),Nt=c(),l=r("span"),w=p("Training loop"),ye=c(),Je=r("p"),xt=p("To keep track of your training progress, use the "),It=r("a"),la=p("tqdm"),Xe=p(" library to add a progress bar over the number of training steps:"),Pe=c(),$(i.$$.fragment),a=c(),b=r("h3"),Se=r("a"),pa=r("span"),$(Dt.$$.fragment),Gt=c(),ha=r("span"),fa=p("Evaluate"),ra=c(),lt=r("p"),Kt=p("Just like how you added an evaluation function to "),oa=r("a"),ya=p("Trainer"),Wt=p(", you need to do the same when you write your own training loop. But instead of calculating and reporting the metric at the end of each epoch, this time you\u2019ll accumulate all the batches with "),Rt=r("code"),wa=p("add_batch"),ga=p(" and calculate the metric at the very end."),da=c(),$(ut.$$.fragment),this.h()},l(s){M(y.$$.fragment,s),E=m(s),u=o(s,"P",{});var g=n(u);v=o(g,"A",{href:!0});var te=n(v);J=h(te,"Trainer"),te.forEach(e),Z=h(g," takes care of the training loop and allows you to fine-tune a model in a single line of code. For users who prefer to write their own training loop, you can also fine-tune a \u{1F917} Transformers model in native PyTorch."),g.forEach(e),z=m(s),I=o(s,"P",{});var ia=n(I);He=h(ia,"At this point, you may need to restart your notebook or execute the following code to free some memory:"),ia.forEach(e),V=m(s),M(G.$$.fragment,s),Qe=m(s),Y=o(s,"P",{});var ca=n(Y);ta=h(ca,"Next, manually postprocess "),wt=o(ca,"CODE",{});var $a=n(wt);R=h($a,"tokenized_dataset"),$a.forEach(e),we=h(ca," to prepare it for training."),ca.forEach(e),jt=m(s),C=o(s,"OL",{});var Pt=n(C);ge=o(Pt,"LI",{});var Ot=n(ge);qe=o(Ot,"P",{});var ma=n(qe);gt=h(ma,"Remove the "),rt=o(ma,"CODE",{});var St=n(rt);kt=h(St,"text"),St.forEach(e),Qt=h(ma," column because the model does not accept raw text as an input:"),ma.forEach(e),Ce=m(Ot),M(ie.$$.fragment,Ot),Ot.forEach(e),Le=m(Pt),q=o(Pt,"LI",{});var Mt=n(q);B=o(Mt,"P",{});var Ht=n(B);ae=h(Ht,"Rename the "),pe=o(Ht,"CODE",{});var ba=n(pe);he=h(ba,"label"),ba.forEach(e),ot=h(Ht," column to "),Be=o(Ht,"CODE",{});var ua=n(Be);Xt=h(ua,"labels"),ua.forEach(e),Ct=h(Ht," because the model expects the argument to be named "),N=o(Ht,"CODE",{});var Ma=n(N);nt=h(Ma,"labels"),Ma.forEach(e),se=h(Ht,":"),Ht.forEach(e),Tt=m(Mt),M(Ze.$$.fragment,Mt),Mt.forEach(e),Fe=m(Pt),be=o(Pt,"LI",{});var ja=n(be);Bt=o(ja,"P",{});var At=n(Bt);ve=h(At,"Set the format of the dataset to return PyTorch tensors instead of lists:"),At.forEach(e),it=m(ja),M(bt.$$.fragment,ja),ja.forEach(e),Pt.forEach(e),_t=m(s),L=o(s,"P",{});var Ea=n(L);De=h(Ea,"Then create a smaller subset of the dataset as previously shown to speed up the fine-tuning:"),Ea.forEach(e),P=m(s),M(x.$$.fragment,s),Ue=m(s),F=o(s,"H3",{class:!0});var ka=n(F);pt=o(ka,"A",{id:!0,class:!0,href:!0});var Ja=n(pt);ht=o(Ja,"SPAN",{});var Ta=n(ht);M(le.$$.fragment,Ta),Ta.forEach(e),Ja.forEach(e),fe=m(ka),U=o(ka,"SPAN",{});var _a=n(U);de=h(_a,"DataLoader"),_a.forEach(e),ka.forEach(e),qt=m(s),Ie=o(s,"P",{});var va=n(Ie);ze=h(va,"Create a "),vt=o(va,"CODE",{});var Ua=n(vt);ce=h(Ua,"DataLoader"),Ua.forEach(e),ft=h(va," for your training and test datasets so you can iterate over batches of data:"),va.forEach(e),$e=m(s),M(Me.$$.fragment,s),Ve=m(s),re=o(s,"P",{});var Va=n(re);aa=h(Va,"Load your model with the number of expected labels:"),Va.forEach(e),Et=m(s),M(je.$$.fragment,s),Ye=m(s),D=o(s,"H3",{class:!0});var Ia=n(D);K=o(Ia,"A",{id:!0,class:!0,href:!0});var Ya=n(K);Jt=o(Ya,"SPAN",{});var Na=n(Jt);M(ke.$$.fragment,Na),Na.forEach(e),Ya.forEach(e),Ft=m(Ia),A=o(Ia,"SPAN",{});var xa=n(A);Ke=h(xa,"Optimizer and learning rate scheduler"),xa.forEach(e),Ia.forEach(e),X=m(s),me=o(s,"P",{});var Ga=n(me);Oe=h(Ga,"Create an optimizer and learning rate scheduler to fine-tune the model. Let\u2019s use the "),Ge=o(Ga,"A",{href:!0,rel:!0});var Pa=n(Ge);zt=o(Pa,"CODE",{});var Sa=n(zt);et=h(Sa,"AdamW"),Sa.forEach(e),Pa.forEach(e),sa=h(Ga," optimizer from PyTorch:"),Ga.forEach(e),Vt=m(s),M(O.$$.fragment,s),Ne=m(s),We=o(s,"P",{});var Wa=n(We);dt=h(Wa,"Create the default learning rate scheduler from "),S=o(Wa,"A",{href:!0});var Ha=n(S);tt=h(Ha,"Trainer"),Ha.forEach(e),Re=h(Wa,":"),Wa.forEach(e),Te=m(s),M(xe.$$.fragment,s),at=m(s),ct=o(s,"P",{});var Ra=n(ct);Zt=h(Ra,"Lastly, specify "),H=o(Ra,"CODE",{});var Qa=n(H);Yt=h(Qa,"device"),Qa.forEach(e),W=h(Ra," to use a GPU if you have access to one. Otherwise, training on a CPU may take several hours instead of a couple of minutes."),Ra.forEach(e),oe=m(s),M(ue.$$.fragment,s),Ae=m(s),M(_e.$$.fragment,s),mt=m(s),Ut=o(s,"P",{});var qa=n(Ut);Lt=h(qa,"Great, now you are ready to train! \u{1F973}"),qa.forEach(e),ee=m(s),Ee=o(s,"H3",{class:!0});var Aa=n(Ee);ne=o(Aa,"A",{id:!0,class:!0,href:!0});var La=n(ne);st=o(La,"SPAN",{});var Da=n(st);M($t.$$.fragment,Da),Da.forEach(e),La.forEach(e),Nt=m(Aa),l=o(Aa,"SPAN",{});var Ka=n(l);w=h(Ka,"Training loop"),Ka.forEach(e),Aa.forEach(e),ye=m(s),Je=o(s,"P",{});var Xa=n(Je);xt=h(Xa,"To keep track of your training progress, use the "),It=o(Xa,"A",{href:!0,rel:!0});var Oa=n(It);la=h(Oa,"tqdm"),Oa.forEach(e),Xe=h(Xa," library to add a progress bar over the number of training steps:"),Xa.forEach(e),Pe=m(s),M(i.$$.fragment,s),a=m(s),b=o(s,"H3",{class:!0});var Ca=n(b);Se=o(Ca,"A",{id:!0,class:!0,href:!0});var es=n(Se);pa=o(es,"SPAN",{});var ts=n(pa);M(Dt.$$.fragment,ts),ts.forEach(e),es.forEach(e),Gt=m(Ca),ha=o(Ca,"SPAN",{});var as=n(ha);fa=h(as,"Evaluate"),as.forEach(e),Ca.forEach(e),ra=m(s),lt=o(s,"P",{});var Za=n(lt);Kt=h(Za,"Just like how you added an evaluation function to "),oa=o(Za,"A",{href:!0});var ss=n(oa);ya=h(ss,"Trainer"),ss.forEach(e),Wt=h(Za,", you need to do the same when you write your own training loop. But instead of calculating and reporting the metric at the end of each epoch, this time you\u2019ll accumulate all the batches with "),Rt=o(Za,"CODE",{});var ls=n(Rt);wa=h(ls,"add_batch"),ls.forEach(e),ga=h(Za," and calculate the metric at the very end."),Za.forEach(e),da=m(s),M(ut.$$.fragment,s),this.h()},h(){d(v,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),d(pt,"id","dataloader"),d(pt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(pt,"href","#dataloader"),d(F,"class","relative group"),d(K,"id","optimizer-and-learning-rate-scheduler"),d(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(K,"href","#optimizer-and-learning-rate-scheduler"),d(D,"class","relative group"),d(Ge,"href","https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html"),d(Ge,"rel","nofollow"),d(S,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),d(ne,"id","training-loop"),d(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ne,"href","#training-loop"),d(Ee,"class","relative group"),d(It,"href","https://tqdm.github.io/"),d(It,"rel","nofollow"),d(Se,"id","evaluate"),d(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Se,"href","#evaluate"),d(b,"class","relative group"),d(oa,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer")},m(s,g){j(y,s,g),f(s,E,g),f(s,u,g),t(u,v),t(v,J),t(u,Z),f(s,z,g),f(s,I,g),t(I,He),f(s,V,g),j(G,s,g),f(s,Qe,g),f(s,Y,g),t(Y,ta),t(Y,wt),t(wt,R),t(Y,we),f(s,jt,g),f(s,C,g),t(C,ge),t(ge,qe),t(qe,gt),t(qe,rt),t(rt,kt),t(qe,Qt),t(ge,Ce),j(ie,ge,null),t(C,Le),t(C,q),t(q,B),t(B,ae),t(B,pe),t(pe,he),t(B,ot),t(B,Be),t(Be,Xt),t(B,Ct),t(B,N),t(N,nt),t(B,se),t(q,Tt),j(Ze,q,null),t(C,Fe),t(C,be),t(be,Bt),t(Bt,ve),t(be,it),j(bt,be,null),f(s,_t,g),f(s,L,g),t(L,De),f(s,P,g),j(x,s,g),f(s,Ue,g),f(s,F,g),t(F,pt),t(pt,ht),j(le,ht,null),t(F,fe),t(F,U),t(U,de),f(s,qt,g),f(s,Ie,g),t(Ie,ze),t(Ie,vt),t(vt,ce),t(Ie,ft),f(s,$e,g),j(Me,s,g),f(s,Ve,g),f(s,re,g),t(re,aa),f(s,Et,g),j(je,s,g),f(s,Ye,g),f(s,D,g),t(D,K),t(K,Jt),j(ke,Jt,null),t(D,Ft),t(D,A),t(A,Ke),f(s,X,g),f(s,me,g),t(me,Oe),t(me,Ge),t(Ge,zt),t(zt,et),t(me,sa),f(s,Vt,g),j(O,s,g),f(s,Ne,g),f(s,We,g),t(We,dt),t(We,S),t(S,tt),t(We,Re),f(s,Te,g),j(xe,s,g),f(s,at,g),f(s,ct,g),t(ct,Zt),t(ct,H),t(H,Yt),t(ct,W),f(s,oe,g),j(ue,s,g),f(s,Ae,g),j(_e,s,g),f(s,mt,g),f(s,Ut,g),t(Ut,Lt),f(s,ee,g),f(s,Ee,g),t(Ee,ne),t(ne,st),j($t,st,null),t(Ee,Nt),t(Ee,l),t(l,w),f(s,ye,g),f(s,Je,g),t(Je,xt),t(Je,It),t(It,la),t(Je,Xe),f(s,Pe,g),j(i,s,g),f(s,a,g),f(s,b,g),t(b,Se),t(Se,pa),j(Dt,pa,null),t(b,Gt),t(b,ha),t(ha,fa),f(s,ra,g),f(s,lt,g),t(lt,Kt),t(lt,oa),t(oa,ya),t(lt,Wt),t(lt,Rt),t(Rt,wa),t(lt,ga),f(s,da,g),j(ut,s,g),na=!0},p(s,g){const te={};g&2&&(te.$$scope={dirty:g,ctx:s}),_e.$set(te)},i(s){na||(k(y.$$.fragment,s),k(G.$$.fragment,s),k(ie.$$.fragment,s),k(Ze.$$.fragment,s),k(bt.$$.fragment,s),k(x.$$.fragment,s),k(le.$$.fragment,s),k(Me.$$.fragment,s),k(je.$$.fragment,s),k(ke.$$.fragment,s),k(O.$$.fragment,s),k(xe.$$.fragment,s),k(ue.$$.fragment,s),k(_e.$$.fragment,s),k($t.$$.fragment,s),k(i.$$.fragment,s),k(Dt.$$.fragment,s),k(ut.$$.fragment,s),na=!0)},o(s){T(y.$$.fragment,s),T(G.$$.fragment,s),T(ie.$$.fragment,s),T(Ze.$$.fragment,s),T(bt.$$.fragment,s),T(x.$$.fragment,s),T(le.$$.fragment,s),T(Me.$$.fragment,s),T(je.$$.fragment,s),T(ke.$$.fragment,s),T(O.$$.fragment,s),T(xe.$$.fragment,s),T(ue.$$.fragment,s),T(_e.$$.fragment,s),T($t.$$.fragment,s),T(i.$$.fragment,s),T(Dt.$$.fragment,s),T(ut.$$.fragment,s),na=!1},d(s){_(y,s),s&&e(E),s&&e(u),s&&e(z),s&&e(I),s&&e(V),_(G,s),s&&e(Qe),s&&e(Y),s&&e(jt),s&&e(C),_(ie),_(Ze),_(bt),s&&e(_t),s&&e(L),s&&e(P),_(x,s),s&&e(Ue),s&&e(F),_(le),s&&e(qt),s&&e(Ie),s&&e($e),_(Me,s),s&&e(Ve),s&&e(re),s&&e(Et),_(je,s),s&&e(Ye),s&&e(D),_(ke),s&&e(X),s&&e(me),s&&e(Vt),_(O,s),s&&e(Ne),s&&e(We),s&&e(Te),_(xe,s),s&&e(at),s&&e(ct),s&&e(oe),_(ue,s),s&&e(Ae),_(_e,s),s&&e(mt),s&&e(Ut),s&&e(ee),s&&e(Ee),_($t),s&&e(ye),s&&e(Je),s&&e(Pe),_(i,s),s&&e(a),s&&e(b),_(Dt),s&&e(ra),s&&e(lt),s&&e(da),_(ut,s)}}}function vs(yt){let y,E;return y=new za({props:{$$slots:{default:[bs]},$$scope:{ctx:yt}}}),{c(){$(y.$$.fragment)},l(u){M(y.$$.fragment,u)},m(u,v){j(y,u,v),E=!0},p(u,v){const J={};v&2&&(J.$$scope={dirty:v,ctx:u}),y.$set(J)},i(u){E||(k(y.$$.fragment,u),E=!0)},o(u){T(y.$$.fragment,u),E=!1},d(u){_(y,u)}}}function $s(yt){let y,E,u,v,J,Z,z,I,He,V,G,Qe,Y,ta,wt,R,we,jt,C,ge,qe,gt,rt,kt,Qt,Ce,ie,Le,q,B,ae,pe,he,ot,Be,Xt,Ct,N,nt,se,Tt,Ze,Fe,be,Bt,ve,it,bt,_t,L,De,P,x,Ue,F,pt,ht,le,fe,U,de,qt,Ie,ze,vt,ce,ft,$e,Me,Ve,re,aa,Et,je,Ye,D,K,Jt,ke,Ft,A,Ke,X,me,Oe,Ge,zt,et,sa,Vt,O,Ne,We,dt,S,tt,Re,Te,xe,at,ct,Zt,H,Yt,W,oe,ue,Ae,_e,mt,Ut,Lt,ee,Ee,ne,st,$t,Nt;return Z=new ea({}),G=new fs({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/training.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/training.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/training.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/training.ipynb"}]}}),ot=new ea({}),nt=new Ba({props:{id:"_BZearw7f0w"}}),L=new Q({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ5ZWxwX3Jldmlld19mdWxsJTIyKSUwQWRhdGFzZXQlNUIlMjJ0cmFpbiUyMiU1RCU1QjEwMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;yelp_review_full&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">100</span>]
{<span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;My expectations for McDonalds are t rarely high. But for one to still fail so spectacularly...that takes something special!\\\\nThe cashier took my friends\\&#x27;s order, then promptly ignored me. I had to force myself in front of a cashier who opened his register to wait on the person BEHIND me. I waited over five minutes for a gigantic order that included precisely one kid\\&#x27;s meal. After watching two people who ordered after me be handed their food, I asked where mine was. The manager started yelling at the cashiers for \\\\&quot;serving off their orders\\\\&quot; when they didn\\&#x27;t have their food. But neither cashier was anywhere near those controls, and the manager was the one serving food to customers and clearing the boards.\\\\nThe manager was rude when giving me my order. She didn\\&#x27;t make sure that I had everything ON MY RECEIPT, and never even had the decency to apologize that I felt I was getting poor service.\\\\nI\\&#x27;ve eaten at various McDonalds restaurants for over 30 years. I\\&#x27;ve worked at more than one location. I expect bad days, bad moods, and the occasional mistake. But I have yet to have a decent experience at this store. It will remain a place I avoid unless someone in my party needs to avoid illness from low blood sugar. Perhaps I should go back to the racially biased service of Steak n Shake instead!&#x27;</span>}`}}),fe=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIpJTBBJTBBJTBBZGVmJTIwdG9rZW5pemVfZnVuY3Rpb24oZXhhbXBsZXMpJTNBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydGV4dCUyMiU1RCUyQyUyMHBhZGRpbmclM0QlMjJtYXhfbGVuZ3RoJTIyJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUpJTBBJTBBJTBBdG9rZW5pemVkX2RhdGFzZXRzJTIwJTNEJTIwZGF0YXNldC5tYXAodG9rZW5pemVfZnVuY3Rpb24lMkMlMjBiYXRjaGVkJTNEVHJ1ZSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer(examples[<span class="hljs-string">&quot;text&quot;</span>], padding=<span class="hljs-string">&quot;max_length&quot;</span>, truncation=<span class="hljs-literal">True</span>)


<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets = dataset.<span class="hljs-built_in">map</span>(tokenize_function, batched=<span class="hljs-literal">True</span>)`}}),ze=new Q({props:{code:"c21hbGxfdHJhaW5fZGF0YXNldCUyMCUzRCUyMHRva2VuaXplZF9kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVELnNodWZmbGUoc2VlZCUzRDQyKS5zZWxlY3QocmFuZ2UoMTAwMCkpJTBBc21hbGxfZXZhbF9kYXRhc2V0JTIwJTNEJTIwdG9rZW5pemVkX2RhdGFzZXRzJTVCJTIydGVzdCUyMiU1RC5zaHVmZmxlKHNlZWQlM0Q0Mikuc2VsZWN0KHJhbmdlKDEwMDApKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>small_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>small_eval_dataset = tokenized_datasets[<span class="hljs-string">&quot;test&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))`}}),re=new ea({}),ke=new rs({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[ws],pytorch:[ms]},$$scope:{ctx:yt}}}),Ge=new ea({}),O=new rs({props:{pytorch:!0,tensorflow:!1,jax:!1,$$slots:{pytorch:[vs]},$$scope:{ctx:yt}}}),Te=new ea({}),{c(){y=r("meta"),E=c(),u=r("h1"),v=r("a"),J=r("span"),$(Z.$$.fragment),z=c(),I=r("span"),He=p("Fine-tune a pretrained model"),V=c(),$(G.$$.fragment),Qe=c(),Y=r("p"),ta=p("There are significant benefits to using a pretrained model. It reduces computation costs, your carbon footprint, and allows you to use state-of-the-art models without having to train one from scratch. \u{1F917} Transformers provides access to thousands of pretrained models for a wide range of tasks. When you use a pretrained model, you train it on a dataset specific to your task. This is known as fine-tuning, an incredibly powerful training technique. In this tutorial, you will fine-tune a pretrained model with a deep learning framework of your choice:"),wt=c(),R=r("ul"),we=r("li"),jt=p("Fine-tune a pretrained model with \u{1F917} Transformers "),C=r("a"),ge=p("Trainer"),qe=p("."),gt=c(),rt=r("li"),kt=p("Fine-tune a pretrained model in TensorFlow with Keras."),Qt=c(),Ce=r("li"),ie=p("Fine-tune a pretrained model in native PyTorch."),Le=c(),q=r("a"),B=c(),ae=r("h2"),pe=r("a"),he=r("span"),$(ot.$$.fragment),Be=c(),Xt=r("span"),Ct=p("Prepare a dataset"),N=c(),$(nt.$$.fragment),se=c(),Tt=r("p"),Ze=p("Before you can fine-tune a pretrained model, download a dataset and prepare it for training. The previous tutorial showed you how to process data for training, and now you get an opportunity to put those skills to the test!"),Fe=c(),be=r("p"),Bt=p("Begin by loading the "),ve=r("a"),it=p("Yelp Reviews"),bt=p(" dataset:"),_t=c(),$(L.$$.fragment),De=c(),P=r("p"),x=p("As you now know, you need a tokenizer to process the text and include a padding and truncation strategy to handle any variable sequence lengths. To process your dataset in one step, use \u{1F917} Datasets "),Ue=r("a"),F=r("code"),pt=p("map"),ht=p(" method to apply a preprocessing function over the entire dataset:"),le=c(),$(fe.$$.fragment),U=c(),de=r("p"),qt=p("If you like, you can create a smaller subset of the full dataset to fine-tune on to reduce the time it takes:"),Ie=c(),$(ze.$$.fragment),vt=c(),ce=r("a"),ft=c(),$e=r("h2"),Me=r("a"),Ve=r("span"),$(re.$$.fragment),aa=c(),Et=r("span"),je=p("Train"),Ye=c(),D=r("p"),K=p(`At this point, you should follow the section corresponding to the framework you want to use. You can use the links
in the right sidebar to jump to the one you want - and if you want to hide all of the content for a given framework,
just use the button at the top-right of that framework\u2019s block!`),Jt=c(),$(ke.$$.fragment),Ft=c(),A=r("a"),Ke=c(),X=r("h2"),me=r("a"),Oe=r("span"),$(Ge.$$.fragment),zt=c(),et=r("span"),sa=p("Train in native PyTorch"),Vt=c(),$(O.$$.fragment),Ne=c(),We=r("a"),dt=c(),S=r("h2"),tt=r("a"),Re=r("span"),$(Te.$$.fragment),xe=c(),at=r("span"),ct=p("Additional resources"),Zt=c(),H=r("p"),Yt=p("For more fine-tuning examples, refer to:"),W=c(),oe=r("ul"),ue=r("li"),Ae=r("p"),_e=r("a"),mt=p("\u{1F917} Transformers Examples"),Ut=p(` includes scripts
to train common NLP tasks in PyTorch and TensorFlow.`),Lt=c(),ee=r("li"),Ee=r("p"),ne=r("a"),st=p("\u{1F917} Transformers Notebooks"),$t=p(" contains various notebooks on how to fine-tune a model for specific tasks in PyTorch and TensorFlow."),this.h()},l(l){const w=ps('[data-svelte="svelte-1phssyn"]',document.head);y=o(w,"META",{name:!0,content:!0}),w.forEach(e),E=m(l),u=o(l,"H1",{class:!0});var ye=n(u);v=o(ye,"A",{id:!0,class:!0,href:!0});var Je=n(v);J=o(Je,"SPAN",{});var xt=n(J);M(Z.$$.fragment,xt),xt.forEach(e),Je.forEach(e),z=m(ye),I=o(ye,"SPAN",{});var It=n(I);He=h(It,"Fine-tune a pretrained model"),It.forEach(e),ye.forEach(e),V=m(l),M(G.$$.fragment,l),Qe=m(l),Y=o(l,"P",{});var la=n(Y);ta=h(la,"There are significant benefits to using a pretrained model. It reduces computation costs, your carbon footprint, and allows you to use state-of-the-art models without having to train one from scratch. \u{1F917} Transformers provides access to thousands of pretrained models for a wide range of tasks. When you use a pretrained model, you train it on a dataset specific to your task. This is known as fine-tuning, an incredibly powerful training technique. In this tutorial, you will fine-tune a pretrained model with a deep learning framework of your choice:"),la.forEach(e),wt=m(l),R=o(l,"UL",{});var Xe=n(R);we=o(Xe,"LI",{});var Pe=n(we);jt=h(Pe,"Fine-tune a pretrained model with \u{1F917} Transformers "),C=o(Pe,"A",{href:!0});var i=n(C);ge=h(i,"Trainer"),i.forEach(e),qe=h(Pe,"."),Pe.forEach(e),gt=m(Xe),rt=o(Xe,"LI",{});var a=n(rt);kt=h(a,"Fine-tune a pretrained model in TensorFlow with Keras."),a.forEach(e),Qt=m(Xe),Ce=o(Xe,"LI",{});var b=n(Ce);ie=h(b,"Fine-tune a pretrained model in native PyTorch."),b.forEach(e),Xe.forEach(e),Le=m(l),q=o(l,"A",{id:!0}),n(q).forEach(e),B=m(l),ae=o(l,"H2",{class:!0});var Se=n(ae);pe=o(Se,"A",{id:!0,class:!0,href:!0});var pa=n(pe);he=o(pa,"SPAN",{});var Dt=n(he);M(ot.$$.fragment,Dt),Dt.forEach(e),pa.forEach(e),Be=m(Se),Xt=o(Se,"SPAN",{});var Gt=n(Xt);Ct=h(Gt,"Prepare a dataset"),Gt.forEach(e),Se.forEach(e),N=m(l),M(nt.$$.fragment,l),se=m(l),Tt=o(l,"P",{});var ha=n(Tt);Ze=h(ha,"Before you can fine-tune a pretrained model, download a dataset and prepare it for training. The previous tutorial showed you how to process data for training, and now you get an opportunity to put those skills to the test!"),ha.forEach(e),Fe=m(l),be=o(l,"P",{});var fa=n(be);Bt=h(fa,"Begin by loading the "),ve=o(fa,"A",{href:!0,rel:!0});var ra=n(ve);it=h(ra,"Yelp Reviews"),ra.forEach(e),bt=h(fa," dataset:"),fa.forEach(e),_t=m(l),M(L.$$.fragment,l),De=m(l),P=o(l,"P",{});var lt=n(P);x=h(lt,"As you now know, you need a tokenizer to process the text and include a padding and truncation strategy to handle any variable sequence lengths. To process your dataset in one step, use \u{1F917} Datasets "),Ue=o(lt,"A",{href:!0,rel:!0});var Kt=n(Ue);F=o(Kt,"CODE",{});var oa=n(F);pt=h(oa,"map"),oa.forEach(e),Kt.forEach(e),ht=h(lt," method to apply a preprocessing function over the entire dataset:"),lt.forEach(e),le=m(l),M(fe.$$.fragment,l),U=m(l),de=o(l,"P",{});var ya=n(de);qt=h(ya,"If you like, you can create a smaller subset of the full dataset to fine-tune on to reduce the time it takes:"),ya.forEach(e),Ie=m(l),M(ze.$$.fragment,l),vt=m(l),ce=o(l,"A",{id:!0}),n(ce).forEach(e),ft=m(l),$e=o(l,"H2",{class:!0});var Wt=n($e);Me=o(Wt,"A",{id:!0,class:!0,href:!0});var Rt=n(Me);Ve=o(Rt,"SPAN",{});var wa=n(Ve);M(re.$$.fragment,wa),wa.forEach(e),Rt.forEach(e),aa=m(Wt),Et=o(Wt,"SPAN",{});var ga=n(Et);je=h(ga,"Train"),ga.forEach(e),Wt.forEach(e),Ye=m(l),D=o(l,"P",{});var da=n(D);K=h(da,`At this point, you should follow the section corresponding to the framework you want to use. You can use the links
in the right sidebar to jump to the one you want - and if you want to hide all of the content for a given framework,
just use the button at the top-right of that framework\u2019s block!`),da.forEach(e),Jt=m(l),M(ke.$$.fragment,l),Ft=m(l),A=o(l,"A",{id:!0}),n(A).forEach(e),Ke=m(l),X=o(l,"H2",{class:!0});var ut=n(X);me=o(ut,"A",{id:!0,class:!0,href:!0});var na=n(me);Oe=o(na,"SPAN",{});var s=n(Oe);M(Ge.$$.fragment,s),s.forEach(e),na.forEach(e),zt=m(ut),et=o(ut,"SPAN",{});var g=n(et);sa=h(g,"Train in native PyTorch"),g.forEach(e),ut.forEach(e),Vt=m(l),M(O.$$.fragment,l),Ne=m(l),We=o(l,"A",{id:!0}),n(We).forEach(e),dt=m(l),S=o(l,"H2",{class:!0});var te=n(S);tt=o(te,"A",{id:!0,class:!0,href:!0});var ia=n(tt);Re=o(ia,"SPAN",{});var ca=n(Re);M(Te.$$.fragment,ca),ca.forEach(e),ia.forEach(e),xe=m(te),at=o(te,"SPAN",{});var $a=n(at);ct=h($a,"Additional resources"),$a.forEach(e),te.forEach(e),Zt=m(l),H=o(l,"P",{});var Pt=n(H);Yt=h(Pt,"For more fine-tuning examples, refer to:"),Pt.forEach(e),W=m(l),oe=o(l,"UL",{});var Ot=n(oe);ue=o(Ot,"LI",{});var ma=n(ue);Ae=o(ma,"P",{});var St=n(Ae);_e=o(St,"A",{href:!0,rel:!0});var Mt=n(_e);mt=h(Mt,"\u{1F917} Transformers Examples"),Mt.forEach(e),Ut=h(St,` includes scripts
to train common NLP tasks in PyTorch and TensorFlow.`),St.forEach(e),ma.forEach(e),Lt=m(Ot),ee=o(Ot,"LI",{});var Ht=n(ee);Ee=o(Ht,"P",{});var ba=n(Ee);ne=o(ba,"A",{href:!0});var ua=n(ne);st=h(ua,"\u{1F917} Transformers Notebooks"),ua.forEach(e),$t=h(ba," contains various notebooks on how to fine-tune a model for specific tasks in PyTorch and TensorFlow."),ba.forEach(e),Ht.forEach(e),Ot.forEach(e),this.h()},h(){d(y,"name","hf:doc:metadata"),d(y,"content",JSON.stringify(Ms)),d(v,"id","finetune-a-pretrained-model"),d(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(v,"href","#finetune-a-pretrained-model"),d(u,"class","relative group"),d(C,"href","/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.Trainer"),d(q,"id","data-processing"),d(pe,"id","prepare-a-dataset"),d(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(pe,"href","#prepare-a-dataset"),d(ae,"class","relative group"),d(ve,"href","https://huggingface.co/datasets/yelp_review_full"),d(ve,"rel","nofollow"),d(Ue,"href","https://huggingface.co/docs/datasets/process.html#map"),d(Ue,"rel","nofollow"),d(ce,"id","trainer"),d(Me,"id","train"),d(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Me,"href","#train"),d($e,"class","relative group"),d(A,"id","pytorch_native"),d(me,"id","train-in-native-pytorch"),d(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(me,"href","#train-in-native-pytorch"),d(X,"class","relative group"),d(We,"id","additional-resources"),d(tt,"id","additional-resources"),d(tt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(tt,"href","#additional-resources"),d(S,"class","relative group"),d(_e,"href","https://github.com/huggingface/transformers/tree/main/examples"),d(_e,"rel","nofollow"),d(ne,"href","notebooks")},m(l,w){t(document.head,y),f(l,E,w),f(l,u,w),t(u,v),t(v,J),j(Z,J,null),t(u,z),t(u,I),t(I,He),f(l,V,w),j(G,l,w),f(l,Qe,w),f(l,Y,w),t(Y,ta),f(l,wt,w),f(l,R,w),t(R,we),t(we,jt),t(we,C),t(C,ge),t(we,qe),t(R,gt),t(R,rt),t(rt,kt),t(R,Qt),t(R,Ce),t(Ce,ie),f(l,Le,w),f(l,q,w),f(l,B,w),f(l,ae,w),t(ae,pe),t(pe,he),j(ot,he,null),t(ae,Be),t(ae,Xt),t(Xt,Ct),f(l,N,w),j(nt,l,w),f(l,se,w),f(l,Tt,w),t(Tt,Ze),f(l,Fe,w),f(l,be,w),t(be,Bt),t(be,ve),t(ve,it),t(be,bt),f(l,_t,w),j(L,l,w),f(l,De,w),f(l,P,w),t(P,x),t(P,Ue),t(Ue,F),t(F,pt),t(P,ht),f(l,le,w),j(fe,l,w),f(l,U,w),f(l,de,w),t(de,qt),f(l,Ie,w),j(ze,l,w),f(l,vt,w),f(l,ce,w),f(l,ft,w),f(l,$e,w),t($e,Me),t(Me,Ve),j(re,Ve,null),t($e,aa),t($e,Et),t(Et,je),f(l,Ye,w),f(l,D,w),t(D,K),f(l,Jt,w),j(ke,l,w),f(l,Ft,w),f(l,A,w),f(l,Ke,w),f(l,X,w),t(X,me),t(me,Oe),j(Ge,Oe,null),t(X,zt),t(X,et),t(et,sa),f(l,Vt,w),j(O,l,w),f(l,Ne,w),f(l,We,w),f(l,dt,w),f(l,S,w),t(S,tt),t(tt,Re),j(Te,Re,null),t(S,xe),t(S,at),t(at,ct),f(l,Zt,w),f(l,H,w),t(H,Yt),f(l,W,w),f(l,oe,w),t(oe,ue),t(ue,Ae),t(Ae,_e),t(_e,mt),t(Ae,Ut),t(oe,Lt),t(oe,ee),t(ee,Ee),t(Ee,ne),t(ne,st),t(Ee,$t),Nt=!0},p(l,[w]){const ye={};w&2&&(ye.$$scope={dirty:w,ctx:l}),ke.$set(ye);const Je={};w&2&&(Je.$$scope={dirty:w,ctx:l}),O.$set(Je)},i(l){Nt||(k(Z.$$.fragment,l),k(G.$$.fragment,l),k(ot.$$.fragment,l),k(nt.$$.fragment,l),k(L.$$.fragment,l),k(fe.$$.fragment,l),k(ze.$$.fragment,l),k(re.$$.fragment,l),k(ke.$$.fragment,l),k(Ge.$$.fragment,l),k(O.$$.fragment,l),k(Te.$$.fragment,l),Nt=!0)},o(l){T(Z.$$.fragment,l),T(G.$$.fragment,l),T(ot.$$.fragment,l),T(nt.$$.fragment,l),T(L.$$.fragment,l),T(fe.$$.fragment,l),T(ze.$$.fragment,l),T(re.$$.fragment,l),T(ke.$$.fragment,l),T(Ge.$$.fragment,l),T(O.$$.fragment,l),T(Te.$$.fragment,l),Nt=!1},d(l){e(y),l&&e(E),l&&e(u),_(Z),l&&e(V),_(G,l),l&&e(Qe),l&&e(Y),l&&e(wt),l&&e(R),l&&e(Le),l&&e(q),l&&e(B),l&&e(ae),_(ot),l&&e(N),_(nt,l),l&&e(se),l&&e(Tt),l&&e(Fe),l&&e(be),l&&e(_t),_(L,l),l&&e(De),l&&e(P),l&&e(le),_(fe,l),l&&e(U),l&&e(de),l&&e(Ie),_(ze,l),l&&e(vt),l&&e(ce),l&&e(ft),l&&e($e),_(re),l&&e(Ye),l&&e(D),l&&e(Jt),_(ke,l),l&&e(Ft),l&&e(A),l&&e(Ke),l&&e(X),_(Ge),l&&e(Vt),_(O,l),l&&e(Ne),l&&e(We),l&&e(dt),l&&e(S),_(Te),l&&e(Zt),l&&e(H),l&&e(W),l&&e(oe)}}}const Ms={local:"finetune-a-pretrained-model",sections:[{local:"prepare-a-dataset",title:"Prepare a dataset"},{local:"train",title:"Train"},{local:"train-with-pytorch-trainer",sections:[{local:"training-hyperparameters",title:"Training hyperparameters"},{local:"evaluate",title:"Evaluate"},{local:"trainer",title:"Trainer"}],title:"Train with PyTorch Trainer"},{local:"train-a-tensorflow-model-with-keras",sections:[{local:"loading-data-for-keras",title:"Loading data for Keras"},{local:"loading-data-as-a-tfdatadataset",title:"Loading data as a tf.data.Dataset"}],title:"Train a TensorFlow model with Keras"},{local:"train-in-native-pytorch",sections:[{local:"dataloader",title:"DataLoader"},{local:"optimizer-and-learning-rate-scheduler",title:"Optimizer and learning rate scheduler"},{local:"training-loop",title:"Training loop"},{local:"evaluate",title:"Evaluate"}],title:"Train in native PyTorch"},{local:"additional-resources",title:"Additional resources"}],title:"Fine-tune a pretrained model"};function js(yt){return hs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Is extends os{constructor(y){super();ns(this,y,js,$s,is,{})}}export{Is as default,Ms as metadata};
