import{S as Pl,i as $l,s as Al,e as o,k as p,w as Gt,t as n,M as Ll,c as a,d as l,m as d,a as r,x as St,h as s,b as i,G as e,g as h,y as Ut,L as Il,q as Bt,o as Nt,B as Ot,v as Cl}from"../../chunks/vendor-hf-doc-builder.js";import{I as kl}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as Gl}from"../../chunks/CodeBlock-hf-doc-builder.js";function Sl(Zt){let w,_e,E,b,Q,A,Oe,H,Ze,we,y,x,X,L,Me,Y,Re,Ee,T,We,I,Je,je,ye,J,De,be,C,xe,j,qe,Te,u,z,G,Fe,S,Ve,Qe,He,K,ee,Xe,Ye,te,le,ze,Ke,oe,ae,et,tt,re,v,lt,ne,ot,at,se,rt,nt,ie,st,it,ke,k,ft,U,ht,pt,Pe,D,dt,$e,c,fe,he,B,ut,ct,pe,de,N,vt,mt,ue,ce,O,gt,_t,ve,me,Z,wt,Et,ge,q,M,yt,bt,Ae,P,xt,F,Tt,kt,Le,m,Pt,R,$t,At,W,Lt,It,Ie;return A=new kl({}),L=new kl({}),C=new Gl({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBJTBBbW9kZWwlMjAlM0QlMjBUNUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbi5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGdDUtdjFfMS1iYXNlJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5ForConditionalGeneration

<span class="hljs-meta">&gt;&gt;&gt; </span>model = T5ForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/t5-v1_1-base&quot;</span>)`}}),{c(){w=o("meta"),_e=p(),E=o("h1"),b=o("a"),Q=o("span"),Gt(A.$$.fragment),Oe=p(),H=o("span"),Ze=n("T5v1.1"),we=p(),y=o("h2"),x=o("a"),X=o("span"),Gt(L.$$.fragment),Me=p(),Y=o("span"),Re=n("Overview"),Ee=p(),T=o("p"),We=n("T5v1.1 was released in the "),I=o("a"),Je=n("google-research/text-to-text-transfer-transformer"),je=n(`
repository by Colin Raffel et al. It\u2019s an improved version of the original T5 model.`),ye=p(),J=o("p"),De=n("One can directly plug in the weights of T5v1.1 into a T5 model, like so:"),be=p(),Gt(C.$$.fragment),xe=p(),j=o("p"),qe=n("T5 Version 1.1 includes the following improvements compared to the original T5 model:"),Te=p(),u=o("ul"),z=o("li"),G=o("p"),Fe=n("GEGLU activation in the feed-forward hidden layer, rather than ReLU. See "),S=o("a"),Ve=n("this paper"),Qe=n("."),He=p(),K=o("li"),ee=o("p"),Xe=n("Dropout was turned off in pre-training (quality win). Dropout should be re-enabled during fine-tuning."),Ye=p(),te=o("li"),le=o("p"),ze=n("Pre-trained on C4 only without mixing in the downstream tasks."),Ke=p(),oe=o("li"),ae=o("p"),et=n("No parameter sharing between the embedding and classifier layer."),tt=p(),re=o("li"),v=o("p"),lt=n("\u201Cxl\u201D and \u201Cxxl\u201D replace \u201C3B\u201D and \u201C11B\u201D. The model shapes are a bit different - larger "),ne=o("code"),ot=n("d_model"),at=n(` and smaller
`),se=o("code"),rt=n("num_heads"),nt=n(" and "),ie=o("code"),st=n("d_ff"),it=n("."),ke=p(),k=o("p"),ft=n("Note: T5 Version 1.1 was only pre-trained on "),U=o("a"),ht=n("C4"),pt=n(` excluding any supervised
training. Therefore, this model has to be fine-tuned before it is usable on a downstream task, unlike the original T5
model. Since t5v1.1 was pre-trained unsupervisedly, there\u2019s no real advantage to using a task prefix during single-task
fine-tuning. If you are doing multi-task fine-tuning, you should use a prefix.`),Pe=p(),D=o("p"),dt=n("Google has released the following variants:"),$e=p(),c=o("ul"),fe=o("li"),he=o("p"),B=o("a"),ut=n("google/t5-v1_1-small"),ct=p(),pe=o("li"),de=o("p"),N=o("a"),vt=n("google/t5-v1_1-base"),mt=p(),ue=o("li"),ce=o("p"),O=o("a"),gt=n("google/t5-v1_1-large"),_t=p(),ve=o("li"),me=o("p"),Z=o("a"),wt=n("google/t5-v1_1-xl"),Et=p(),ge=o("li"),q=o("p"),M=o("a"),yt=n("google/t5-v1_1-xxl"),bt=n("."),Ae=p(),P=o("p"),xt=n("One can refer to "),F=o("a"),Tt=n("T5\u2019s documentation page"),kt=n(" for all tips, code examples and notebooks."),Le=p(),m=o("p"),Pt=n("This model was contributed by "),R=o("a"),$t=n("patrickvonplaten"),At=n(`. The original code can be
found `),W=o("a"),Lt=n("here"),It=n("."),this.h()},l(t){const f=Ll('[data-svelte="svelte-1phssyn"]',document.head);w=a(f,"META",{name:!0,content:!0}),f.forEach(l),_e=d(t),E=a(t,"H1",{class:!0});var Ce=r(E);b=a(Ce,"A",{id:!0,class:!0,href:!0});var Mt=r(b);Q=a(Mt,"SPAN",{});var Rt=r(Q);St(A.$$.fragment,Rt),Rt.forEach(l),Mt.forEach(l),Oe=d(Ce),H=a(Ce,"SPAN",{});var Wt=r(H);Ze=s(Wt,"T5v1.1"),Wt.forEach(l),Ce.forEach(l),we=d(t),y=a(t,"H2",{class:!0});var Ge=r(y);x=a(Ge,"A",{id:!0,class:!0,href:!0});var Jt=r(x);X=a(Jt,"SPAN",{});var jt=r(X);St(L.$$.fragment,jt),jt.forEach(l),Jt.forEach(l),Me=d(Ge),Y=a(Ge,"SPAN",{});var Dt=r(Y);Re=s(Dt,"Overview"),Dt.forEach(l),Ge.forEach(l),Ee=d(t),T=a(t,"P",{});var Se=r(T);We=s(Se,"T5v1.1 was released in the "),I=a(Se,"A",{href:!0,rel:!0});var qt=r(I);Je=s(qt,"google-research/text-to-text-transfer-transformer"),qt.forEach(l),je=s(Se,`
repository by Colin Raffel et al. It\u2019s an improved version of the original T5 model.`),Se.forEach(l),ye=d(t),J=a(t,"P",{});var Ft=r(J);De=s(Ft,"One can directly plug in the weights of T5v1.1 into a T5 model, like so:"),Ft.forEach(l),be=d(t),St(C.$$.fragment,t),xe=d(t),j=a(t,"P",{});var Vt=r(j);qe=s(Vt,"T5 Version 1.1 includes the following improvements compared to the original T5 model:"),Vt.forEach(l),Te=d(t),u=a(t,"UL",{});var g=r(u);z=a(g,"LI",{});var Qt=r(z);G=a(Qt,"P",{});var Ue=r(G);Fe=s(Ue,"GEGLU activation in the feed-forward hidden layer, rather than ReLU. See "),S=a(Ue,"A",{href:!0,rel:!0});var Ht=r(S);Ve=s(Ht,"this paper"),Ht.forEach(l),Qe=s(Ue,"."),Ue.forEach(l),Qt.forEach(l),He=d(g),K=a(g,"LI",{});var Xt=r(K);ee=a(Xt,"P",{});var Yt=r(ee);Xe=s(Yt,"Dropout was turned off in pre-training (quality win). Dropout should be re-enabled during fine-tuning."),Yt.forEach(l),Xt.forEach(l),Ye=d(g),te=a(g,"LI",{});var zt=r(te);le=a(zt,"P",{});var Kt=r(le);ze=s(Kt,"Pre-trained on C4 only without mixing in the downstream tasks."),Kt.forEach(l),zt.forEach(l),Ke=d(g),oe=a(g,"LI",{});var el=r(oe);ae=a(el,"P",{});var tl=r(ae);et=s(tl,"No parameter sharing between the embedding and classifier layer."),tl.forEach(l),el.forEach(l),tt=d(g),re=a(g,"LI",{});var ll=r(re);v=a(ll,"P",{});var $=r(v);lt=s($,"\u201Cxl\u201D and \u201Cxxl\u201D replace \u201C3B\u201D and \u201C11B\u201D. The model shapes are a bit different - larger "),ne=a($,"CODE",{});var ol=r(ne);ot=s(ol,"d_model"),ol.forEach(l),at=s($,` and smaller
`),se=a($,"CODE",{});var al=r(se);rt=s(al,"num_heads"),al.forEach(l),nt=s($," and "),ie=a($,"CODE",{});var rl=r(ie);st=s(rl,"d_ff"),rl.forEach(l),it=s($,"."),$.forEach(l),ll.forEach(l),g.forEach(l),ke=d(t),k=a(t,"P",{});var Be=r(k);ft=s(Be,"Note: T5 Version 1.1 was only pre-trained on "),U=a(Be,"A",{href:!0,rel:!0});var nl=r(U);ht=s(nl,"C4"),nl.forEach(l),pt=s(Be,` excluding any supervised
training. Therefore, this model has to be fine-tuned before it is usable on a downstream task, unlike the original T5
model. Since t5v1.1 was pre-trained unsupervisedly, there\u2019s no real advantage to using a task prefix during single-task
fine-tuning. If you are doing multi-task fine-tuning, you should use a prefix.`),Be.forEach(l),Pe=d(t),D=a(t,"P",{});var sl=r(D);dt=s(sl,"Google has released the following variants:"),sl.forEach(l),$e=d(t),c=a(t,"UL",{});var _=r(c);fe=a(_,"LI",{});var il=r(fe);he=a(il,"P",{});var fl=r(he);B=a(fl,"A",{href:!0,rel:!0});var hl=r(B);ut=s(hl,"google/t5-v1_1-small"),hl.forEach(l),fl.forEach(l),il.forEach(l),ct=d(_),pe=a(_,"LI",{});var pl=r(pe);de=a(pl,"P",{});var dl=r(de);N=a(dl,"A",{href:!0,rel:!0});var ul=r(N);vt=s(ul,"google/t5-v1_1-base"),ul.forEach(l),dl.forEach(l),pl.forEach(l),mt=d(_),ue=a(_,"LI",{});var cl=r(ue);ce=a(cl,"P",{});var vl=r(ce);O=a(vl,"A",{href:!0,rel:!0});var ml=r(O);gt=s(ml,"google/t5-v1_1-large"),ml.forEach(l),vl.forEach(l),cl.forEach(l),_t=d(_),ve=a(_,"LI",{});var gl=r(ve);me=a(gl,"P",{});var _l=r(me);Z=a(_l,"A",{href:!0,rel:!0});var wl=r(Z);wt=s(wl,"google/t5-v1_1-xl"),wl.forEach(l),_l.forEach(l),gl.forEach(l),Et=d(_),ge=a(_,"LI",{});var El=r(ge);q=a(El,"P",{});var Ct=r(q);M=a(Ct,"A",{href:!0,rel:!0});var yl=r(M);yt=s(yl,"google/t5-v1_1-xxl"),yl.forEach(l),bt=s(Ct,"."),Ct.forEach(l),El.forEach(l),_.forEach(l),Ae=d(t),P=a(t,"P",{});var Ne=r(P);xt=s(Ne,"One can refer to "),F=a(Ne,"A",{href:!0});var bl=r(F);Tt=s(bl,"T5\u2019s documentation page"),bl.forEach(l),kt=s(Ne," for all tips, code examples and notebooks."),Ne.forEach(l),Le=d(t),m=a(t,"P",{});var V=r(m);Pt=s(V,"This model was contributed by "),R=a(V,"A",{href:!0,rel:!0});var xl=r(R);$t=s(xl,"patrickvonplaten"),xl.forEach(l),At=s(V,`. The original code can be
found `),W=a(V,"A",{href:!0,rel:!0});var Tl=r(W);Lt=s(Tl,"here"),Tl.forEach(l),It=s(V,"."),V.forEach(l),this.h()},h(){i(w,"name","hf:doc:metadata"),i(w,"content",JSON.stringify(Ul)),i(b,"id","t5v11"),i(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(b,"href","#t5v11"),i(E,"class","relative group"),i(x,"id","overview"),i(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(x,"href","#overview"),i(y,"class","relative group"),i(I,"href","https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511"),i(I,"rel","nofollow"),i(S,"href","https://arxiv.org/abs/2002.05202"),i(S,"rel","nofollow"),i(U,"href","https://huggingface.co/datasets/c4"),i(U,"rel","nofollow"),i(B,"href","https://huggingface.co/google/t5-v1_1-small"),i(B,"rel","nofollow"),i(N,"href","https://huggingface.co/google/t5-v1_1-base"),i(N,"rel","nofollow"),i(O,"href","https://huggingface.co/google/t5-v1_1-large"),i(O,"rel","nofollow"),i(Z,"href","https://huggingface.co/google/t5-v1_1-xl"),i(Z,"rel","nofollow"),i(M,"href","https://huggingface.co/google/t5-v1_1-xxl"),i(M,"rel","nofollow"),i(F,"href","t5"),i(R,"href","https://huggingface.co/patrickvonplaten"),i(R,"rel","nofollow"),i(W,"href","https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511"),i(W,"rel","nofollow")},m(t,f){e(document.head,w),h(t,_e,f),h(t,E,f),e(E,b),e(b,Q),Ut(A,Q,null),e(E,Oe),e(E,H),e(H,Ze),h(t,we,f),h(t,y,f),e(y,x),e(x,X),Ut(L,X,null),e(y,Me),e(y,Y),e(Y,Re),h(t,Ee,f),h(t,T,f),e(T,We),e(T,I),e(I,Je),e(T,je),h(t,ye,f),h(t,J,f),e(J,De),h(t,be,f),Ut(C,t,f),h(t,xe,f),h(t,j,f),e(j,qe),h(t,Te,f),h(t,u,f),e(u,z),e(z,G),e(G,Fe),e(G,S),e(S,Ve),e(G,Qe),e(u,He),e(u,K),e(K,ee),e(ee,Xe),e(u,Ye),e(u,te),e(te,le),e(le,ze),e(u,Ke),e(u,oe),e(oe,ae),e(ae,et),e(u,tt),e(u,re),e(re,v),e(v,lt),e(v,ne),e(ne,ot),e(v,at),e(v,se),e(se,rt),e(v,nt),e(v,ie),e(ie,st),e(v,it),h(t,ke,f),h(t,k,f),e(k,ft),e(k,U),e(U,ht),e(k,pt),h(t,Pe,f),h(t,D,f),e(D,dt),h(t,$e,f),h(t,c,f),e(c,fe),e(fe,he),e(he,B),e(B,ut),e(c,ct),e(c,pe),e(pe,de),e(de,N),e(N,vt),e(c,mt),e(c,ue),e(ue,ce),e(ce,O),e(O,gt),e(c,_t),e(c,ve),e(ve,me),e(me,Z),e(Z,wt),e(c,Et),e(c,ge),e(ge,q),e(q,M),e(M,yt),e(q,bt),h(t,Ae,f),h(t,P,f),e(P,xt),e(P,F),e(F,Tt),e(P,kt),h(t,Le,f),h(t,m,f),e(m,Pt),e(m,R),e(R,$t),e(m,At),e(m,W),e(W,Lt),e(m,It),Ie=!0},p:Il,i(t){Ie||(Bt(A.$$.fragment,t),Bt(L.$$.fragment,t),Bt(C.$$.fragment,t),Ie=!0)},o(t){Nt(A.$$.fragment,t),Nt(L.$$.fragment,t),Nt(C.$$.fragment,t),Ie=!1},d(t){l(w),t&&l(_e),t&&l(E),Ot(A),t&&l(we),t&&l(y),Ot(L),t&&l(Ee),t&&l(T),t&&l(ye),t&&l(J),t&&l(be),Ot(C,t),t&&l(xe),t&&l(j),t&&l(Te),t&&l(u),t&&l(ke),t&&l(k),t&&l(Pe),t&&l(D),t&&l($e),t&&l(c),t&&l(Ae),t&&l(P),t&&l(Le),t&&l(m)}}}const Ul={local:"t5v11",sections:[{local:"overview",title:"Overview"}],title:"T5v1.1"};function Bl(Zt){return Cl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ml extends Pl{constructor(w){super();$l(this,w,Bl,Sl,Al,{})}}export{Ml as default,Ul as metadata};
