import{S as tt,i as at,s as rt,e as r,k as d,w as Ie,t as s,M as ot,c as o,d as t,m as v,a as n,x as Ue,h as i,b as h,G as a,g as f,y as ze,q as Le,o as Se,B as qe,v as nt}from"../../chunks/vendor-hf-doc-builder.js";import{T as st}from"../../chunks/Tip-hf-doc-builder.js";import{I as et}from"../../chunks/IconCopyLink-hf-doc-builder.js";function it(K){let u,B,m,c,_,w,k,y;return{c(){u=r("p"),B=s("This model is in maintenance mode only, so we won\u2019t accept any new PRs changing its code."),m=d(),c=r("p"),_=s(`If you run into any issues running this model, please reinstall the last version that supported this model: v4.30.0.
You can do so by running the following command: `),w=r("code"),k=s("pip install -U transformers==4.30.0"),y=s(".")},l(p){u=o(p,"P",{});var g=n(u);B=i(g,"This model is in maintenance mode only, so we won\u2019t accept any new PRs changing its code."),g.forEach(t),m=v(p),c=o(p,"P",{});var b=n(c);_=i(b,`If you run into any issues running this model, please reinstall the last version that supported this model: v4.30.0.
You can do so by running the following command: `),w=o(b,"CODE",{});var A=n(w);k=i(A,"pip install -U transformers==4.30.0"),A.forEach(t),y=i(b,"."),b.forEach(t)},m(p,g){f(p,u,g),a(u,B),f(p,m,g),f(p,c,g),a(c,_),a(c,w),a(w,k),a(c,y)},d(p){p&&t(u),p&&t(m),p&&t(c)}}}function lt(K){let u,B,m,c,_,w,k,y,p,g,b,A,R,$,Y,x,le,j,he,Q,P,fe,O,ce,ue,V,C,pe,X,D,F,me,Z,G,de,ee,E,I,ve,M,we,be,ge,U,Ee,W,Te,_e,ye,z,Re,L,Be,$e,te,T,Pe,S,ke,Ae,q,xe,Oe,ae;return w=new et({}),b=new st({props:{warning:!0,$$slots:{default:[it]},$$scope:{ctx:K}}}),x=new et({}),{c(){u=r("meta"),B=d(),m=r("h1"),c=r("a"),_=r("span"),Ie(w.$$.fragment),k=d(),y=r("span"),p=s("BORT"),g=d(),Ie(b.$$.fragment),A=d(),R=r("h2"),$=r("a"),Y=r("span"),Ie(x.$$.fragment),le=d(),j=r("span"),he=s("Overview"),Q=d(),P=r("p"),fe=s("The BORT model was proposed in "),O=r("a"),ce=s("Optimal Subarchitecture Extraction for BERT"),ue=s(` by
Adrian de Wynter and Daniel J. Perry. It is an optimal subset of architectural parameters for the BERT, which the
authors refer to as \u201CBort\u201D.`),V=d(),C=r("p"),pe=s("The abstract from the paper is the following:"),X=d(),D=r("p"),F=r("em"),me=s(`We extract an optimal subset of architectural parameters for the BERT architecture from Devlin et al. (2018) by
applying recent breakthroughs in algorithms for neural architecture search. This optimal subset, which we refer to as
\u201CBort\u201D, is demonstrably smaller, having an effective (that is, not counting the embedding layer) size of 5.5% the
original BERT-large architecture, and 16% of the net size. Bort is also able to be pretrained in 288 GPU hours, which
is 1.2% of the time required to pretrain the highest-performing BERT parametric architectural variant, RoBERTa-large
(Liu et al., 2019), and about 33% of that of the world-record, in GPU hours, required to train BERT-large on the same
hardware. It is also 7.9x faster on a CPU, as well as being better performing than other compressed variants of the
architecture, and some of the non-compressed variants: it obtains performance improvements of between 0.3% and 31%,
absolute, with respect to BERT-large, on multiple public natural language understanding (NLU) benchmarks.`),Z=d(),G=r("p"),de=s("Tips:"),ee=d(),E=r("ul"),I=r("li"),ve=s("BORT\u2019s model architecture is based on BERT, so one can refer to "),M=r("a"),we=s("BERT\u2019s documentation page"),be=s(` for the
model\u2019s API as well as usage examples.`),ge=d(),U=r("li"),Ee=s("BORT uses the RoBERTa tokenizer instead of the BERT tokenizer, so one can refer to "),W=r("a"),Te=s("RoBERTa\u2019s documentation page"),_e=s(" for the tokenizer\u2019s API as well as usage examples."),ye=d(),z=r("li"),Re=s("BORT requires a specific fine-tuning algorithm, called "),L=r("a"),Be=s("Agora"),$e=s(` ,
that is sadly not open-sourced yet. It would be very useful for the community, if someone tries to implement the
algorithm to make BORT fine-tuning work.`),te=d(),T=r("p"),Pe=s("This model was contributed by "),S=r("a"),ke=s("stefan-it"),Ae=s(". The original code can be found "),q=r("a"),xe=s("here"),Oe=s("."),this.h()},l(e){const l=ot('[data-svelte="svelte-1phssyn"]',document.head);u=o(l,"META",{name:!0,content:!0}),l.forEach(t),B=v(e),m=o(e,"H1",{class:!0});var N=n(m);c=o(N,"A",{id:!0,class:!0,href:!0});var Ne=n(c);_=o(Ne,"SPAN",{});var Ce=n(_);Ue(w.$$.fragment,Ce),Ce.forEach(t),Ne.forEach(t),k=v(N),y=o(N,"SPAN",{});var De=n(y);p=i(De,"BORT"),De.forEach(t),N.forEach(t),g=v(e),Ue(b.$$.fragment,e),A=v(e),R=o(e,"H2",{class:!0});var re=n(R);$=o(re,"A",{id:!0,class:!0,href:!0});var Ge=n($);Y=o(Ge,"SPAN",{});var Me=n(Y);Ue(x.$$.fragment,Me),Me.forEach(t),Ge.forEach(t),le=v(re),j=o(re,"SPAN",{});var We=n(j);he=i(We,"Overview"),We.forEach(t),re.forEach(t),Q=v(e),P=o(e,"P",{});var oe=n(P);fe=i(oe,"The BORT model was proposed in "),O=o(oe,"A",{href:!0,rel:!0});var Je=n(O);ce=i(Je,"Optimal Subarchitecture Extraction for BERT"),Je.forEach(t),ue=i(oe,` by
Adrian de Wynter and Daniel J. Perry. It is an optimal subset of architectural parameters for the BERT, which the
authors refer to as \u201CBort\u201D.`),oe.forEach(t),V=v(e),C=o(e,"P",{});var He=n(C);pe=i(He,"The abstract from the paper is the following:"),He.forEach(t),X=v(e),D=o(e,"P",{});var Ye=n(D);F=o(Ye,"EM",{});var je=n(F);me=i(je,`We extract an optimal subset of architectural parameters for the BERT architecture from Devlin et al. (2018) by
applying recent breakthroughs in algorithms for neural architecture search. This optimal subset, which we refer to as
\u201CBort\u201D, is demonstrably smaller, having an effective (that is, not counting the embedding layer) size of 5.5% the
original BERT-large architecture, and 16% of the net size. Bort is also able to be pretrained in 288 GPU hours, which
is 1.2% of the time required to pretrain the highest-performing BERT parametric architectural variant, RoBERTa-large
(Liu et al., 2019), and about 33% of that of the world-record, in GPU hours, required to train BERT-large on the same
hardware. It is also 7.9x faster on a CPU, as well as being better performing than other compressed variants of the
architecture, and some of the non-compressed variants: it obtains performance improvements of between 0.3% and 31%,
absolute, with respect to BERT-large, on multiple public natural language understanding (NLU) benchmarks.`),je.forEach(t),Ye.forEach(t),Z=v(e),G=o(e,"P",{});var Fe=n(G);de=i(Fe,"Tips:"),Fe.forEach(t),ee=v(e),E=o(e,"UL",{});var J=n(E);I=o(J,"LI",{});var ne=n(I);ve=i(ne,"BORT\u2019s model architecture is based on BERT, so one can refer to "),M=o(ne,"A",{href:!0});var Ke=n(M);we=i(Ke,"BERT\u2019s documentation page"),Ke.forEach(t),be=i(ne,` for the
model\u2019s API as well as usage examples.`),ne.forEach(t),ge=v(J),U=o(J,"LI",{});var se=n(U);Ee=i(se,"BORT uses the RoBERTa tokenizer instead of the BERT tokenizer, so one can refer to "),W=o(se,"A",{href:!0});var Qe=n(W);Te=i(Qe,"RoBERTa\u2019s documentation page"),Qe.forEach(t),_e=i(se," for the tokenizer\u2019s API as well as usage examples."),se.forEach(t),ye=v(J),z=o(J,"LI",{});var ie=n(z);Re=i(ie,"BORT requires a specific fine-tuning algorithm, called "),L=o(ie,"A",{href:!0,rel:!0});var Ve=n(L);Be=i(Ve,"Agora"),Ve.forEach(t),$e=i(ie,` ,
that is sadly not open-sourced yet. It would be very useful for the community, if someone tries to implement the
algorithm to make BORT fine-tuning work.`),ie.forEach(t),J.forEach(t),te=v(e),T=o(e,"P",{});var H=n(T);Pe=i(H,"This model was contributed by "),S=o(H,"A",{href:!0,rel:!0});var Xe=n(S);ke=i(Xe,"stefan-it"),Xe.forEach(t),Ae=i(H,". The original code can be found "),q=o(H,"A",{href:!0,rel:!0});var Ze=n(q);xe=i(Ze,"here"),Ze.forEach(t),Oe=i(H,"."),H.forEach(t),this.h()},h(){h(u,"name","hf:doc:metadata"),h(u,"content",JSON.stringify(ht)),h(c,"id","bort"),h(c,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(c,"href","#bort"),h(m,"class","relative group"),h($,"id","overview"),h($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h($,"href","#overview"),h(R,"class","relative group"),h(O,"href","https://arxiv.org/abs/2010.10499"),h(O,"rel","nofollow"),h(M,"href","bert"),h(W,"href","roberta"),h(L,"href","https://adewynter.github.io/notes/bort_algorithms_and_applications.html#fine-tuning-with-algebraic-topology"),h(L,"rel","nofollow"),h(S,"href","https://huggingface.co/stefan-it"),h(S,"rel","nofollow"),h(q,"href","https://github.com/alexa/bort/"),h(q,"rel","nofollow")},m(e,l){a(document.head,u),f(e,B,l),f(e,m,l),a(m,c),a(c,_),ze(w,_,null),a(m,k),a(m,y),a(y,p),f(e,g,l),ze(b,e,l),f(e,A,l),f(e,R,l),a(R,$),a($,Y),ze(x,Y,null),a(R,le),a(R,j),a(j,he),f(e,Q,l),f(e,P,l),a(P,fe),a(P,O),a(O,ce),a(P,ue),f(e,V,l),f(e,C,l),a(C,pe),f(e,X,l),f(e,D,l),a(D,F),a(F,me),f(e,Z,l),f(e,G,l),a(G,de),f(e,ee,l),f(e,E,l),a(E,I),a(I,ve),a(I,M),a(M,we),a(I,be),a(E,ge),a(E,U),a(U,Ee),a(U,W),a(W,Te),a(U,_e),a(E,ye),a(E,z),a(z,Re),a(z,L),a(L,Be),a(z,$e),f(e,te,l),f(e,T,l),a(T,Pe),a(T,S),a(S,ke),a(T,Ae),a(T,q),a(q,xe),a(T,Oe),ae=!0},p(e,[l]){const N={};l&2&&(N.$$scope={dirty:l,ctx:e}),b.$set(N)},i(e){ae||(Le(w.$$.fragment,e),Le(b.$$.fragment,e),Le(x.$$.fragment,e),ae=!0)},o(e){Se(w.$$.fragment,e),Se(b.$$.fragment,e),Se(x.$$.fragment,e),ae=!1},d(e){t(u),e&&t(B),e&&t(m),qe(w),e&&t(g),qe(b,e),e&&t(A),e&&t(R),qe(x),e&&t(Q),e&&t(P),e&&t(V),e&&t(C),e&&t(X),e&&t(D),e&&t(Z),e&&t(G),e&&t(ee),e&&t(E),e&&t(te),e&&t(T)}}}const ht={local:"bort",sections:[{local:"overview",title:"Overview"}],title:"BORT"};function ft(K){return nt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class mt extends tt{constructor(u){super();at(this,u,ft,lt,rt,{})}}export{mt as default,ht as metadata};
