import{S as Ke,i as je,s as qe,e as r,k as p,w as We,t as s,M as Je,c as o,d as a,m as d,a as l,x as ze,h as n,b as i,G as e,g as c,y as Fe,L as Qe,q as Ve,o as Xe,B as Ye,v as Ze}from"../chunks/vendor-hf-doc-builder.js";import{I as et}from"../chunks/IconCopyLink-hf-doc-builder.js";function tt(Re){let v,U,m,g,$,w,X,D,Y,H,k,Z,K,f,G,ee,_,te,ae,I,re,T,oe,le,M,se,x,ne,ie,C,he,P,fe,j,y,pe,A,de,ce,q,u,N,ue,ve,O,me,ge,L,ye,R,Ee,be,W,E,we,B,_e,Te,J;return w=new et({}),{c(){v=r("meta"),U=p(),m=r("h1"),g=r("a"),$=r("span"),We(w.$$.fragment),X=p(),D=r("span"),Y=s("BERTology"),H=p(),k=r("p"),Z=s(`There is a growing field of study concerned with investigating the inner working of large-scale transformers like BERT
(that some call \u201CBERTology\u201D). Some good examples of this field are:`),K=p(),f=r("ul"),G=r("li"),ee=s(`BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick:
`),_=r("a"),te=s("https://arxiv.org/abs/1905.05950"),ae=p(),I=r("li"),re=s("Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: "),T=r("a"),oe=s("https://arxiv.org/abs/1905.10650"),le=p(),M=r("li"),se=s(`What Does BERT Look At? An Analysis of BERT\u2019s Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D.
Manning: `),x=r("a"),ne=s("https://arxiv.org/abs/1906.04341"),ie=p(),C=r("li"),he=s("CAT-probing: A Metric-based Approach to Interpret How Pre-trained Models for Programming Language Attend Code Structure: "),P=r("a"),fe=s("https://arxiv.org/abs/2210.04633"),j=p(),y=r("p"),pe=s(`In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to
help people access the inner representations, mainly adapted from the great work of Paul Michel
(`),A=r("a"),de=s("https://arxiv.org/abs/1905.10650"),ce=s("):"),q=p(),u=r("ul"),N=r("li"),ue=s("accessing all the hidden-states of BERT/GPT/GPT-2,"),ve=p(),O=r("li"),me=s("accessing all the attention weights for each head of BERT/GPT/GPT-2,"),ge=p(),L=r("li"),ye=s(`retrieving heads output values and gradients to be able to compute head importance score and prune head as explained
in `),R=r("a"),Ee=s("https://arxiv.org/abs/1905.10650"),be=s("."),W=p(),E=r("p"),we=s("To help you understand and use these features, we have added a specific example script: "),B=r("a"),_e=s("bertology.py"),Te=s(` while extract information and prune a model pre-trained on
GLUE.`),this.h()},l(t){const h=Je('[data-svelte="svelte-1phssyn"]',document.head);v=o(h,"META",{name:!0,content:!0}),h.forEach(a),U=d(t),m=o(t,"H1",{class:!0});var z=l(m);g=o(z,"A",{id:!0,class:!0,href:!0});var Be=l(g);$=o(Be,"SPAN",{});var ke=l($);ze(w.$$.fragment,ke),ke.forEach(a),Be.forEach(a),X=d(z),D=o(z,"SPAN",{});var Ge=l(D);Y=n(Ge,"BERTology"),Ge.forEach(a),z.forEach(a),H=d(t),k=o(t,"P",{});var Ie=l(k);Z=n(Ie,`There is a growing field of study concerned with investigating the inner working of large-scale transformers like BERT
(that some call \u201CBERTology\u201D). Some good examples of this field are:`),Ie.forEach(a),K=d(t),f=o(t,"UL",{});var b=l(f);G=o(b,"LI",{});var xe=l(G);ee=n(xe,`BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick:
`),_=o(xe,"A",{href:!0,rel:!0});var Me=l(_);te=n(Me,"https://arxiv.org/abs/1905.05950"),Me.forEach(a),xe.forEach(a),ae=d(b),I=o(b,"LI",{});var Pe=l(I);re=n(Pe,"Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: "),T=o(Pe,"A",{href:!0,rel:!0});var Ce=l(T);oe=n(Ce,"https://arxiv.org/abs/1905.10650"),Ce.forEach(a),Pe.forEach(a),le=d(b),M=o(b,"LI",{});var Ae=l(M);se=n(Ae,`What Does BERT Look At? An Analysis of BERT\u2019s Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D.
Manning: `),x=o(Ae,"A",{href:!0,rel:!0});var Se=l(x);ne=n(Se,"https://arxiv.org/abs/1906.04341"),Se.forEach(a),Ae.forEach(a),ie=d(b),C=o(b,"LI",{});var Le=l(C);he=n(Le,"CAT-probing: A Metric-based Approach to Interpret How Pre-trained Models for Programming Language Attend Code Structure: "),P=o(Le,"A",{href:!0,rel:!0});var $e=l(P);fe=n($e,"https://arxiv.org/abs/2210.04633"),$e.forEach(a),Le.forEach(a),b.forEach(a),j=d(t),y=o(t,"P",{});var F=l(y);pe=n(F,`In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to
help people access the inner representations, mainly adapted from the great work of Paul Michel
(`),A=o(F,"A",{href:!0,rel:!0});var De=l(A);de=n(De,"https://arxiv.org/abs/1905.10650"),De.forEach(a),ce=n(F,"):"),F.forEach(a),q=d(t),u=o(t,"UL",{});var S=l(u);N=o(S,"LI",{});var Ne=l(N);ue=n(Ne,"accessing all the hidden-states of BERT/GPT/GPT-2,"),Ne.forEach(a),ve=d(S),O=o(S,"LI",{});var Oe=l(O);me=n(Oe,"accessing all the attention weights for each head of BERT/GPT/GPT-2,"),Oe.forEach(a),ge=d(S),L=o(S,"LI",{});var Q=l(L);ye=n(Q,`retrieving heads output values and gradients to be able to compute head importance score and prune head as explained
in `),R=o(Q,"A",{href:!0,rel:!0});var Ue=l(R);Ee=n(Ue,"https://arxiv.org/abs/1905.10650"),Ue.forEach(a),be=n(Q,"."),Q.forEach(a),S.forEach(a),W=d(t),E=o(t,"P",{});var V=l(E);we=n(V,"To help you understand and use these features, we have added a specific example script: "),B=o(V,"A",{href:!0,rel:!0});var He=l(B);_e=n(He,"bertology.py"),He.forEach(a),Te=n(V,` while extract information and prune a model pre-trained on
GLUE.`),V.forEach(a),this.h()},h(){i(v,"name","hf:doc:metadata"),i(v,"content",JSON.stringify(at)),i(g,"id","bertology"),i(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(g,"href","#bertology"),i(m,"class","relative group"),i(_,"href","https://arxiv.org/abs/1905.05950"),i(_,"rel","nofollow"),i(T,"href","https://arxiv.org/abs/1905.10650"),i(T,"rel","nofollow"),i(x,"href","https://arxiv.org/abs/1906.04341"),i(x,"rel","nofollow"),i(P,"href","https://arxiv.org/abs/2210.04633"),i(P,"rel","nofollow"),i(A,"href","https://arxiv.org/abs/1905.10650"),i(A,"rel","nofollow"),i(R,"href","https://arxiv.org/abs/1905.10650"),i(R,"rel","nofollow"),i(B,"href","https://github.com/huggingface/transformers/tree/main/examples/research_projects/bertology/run_bertology.py"),i(B,"rel","nofollow")},m(t,h){e(document.head,v),c(t,U,h),c(t,m,h),e(m,g),e(g,$),Fe(w,$,null),e(m,X),e(m,D),e(D,Y),c(t,H,h),c(t,k,h),e(k,Z),c(t,K,h),c(t,f,h),e(f,G),e(G,ee),e(G,_),e(_,te),e(f,ae),e(f,I),e(I,re),e(I,T),e(T,oe),e(f,le),e(f,M),e(M,se),e(M,x),e(x,ne),e(f,ie),e(f,C),e(C,he),e(C,P),e(P,fe),c(t,j,h),c(t,y,h),e(y,pe),e(y,A),e(A,de),e(y,ce),c(t,q,h),c(t,u,h),e(u,N),e(N,ue),e(u,ve),e(u,O),e(O,me),e(u,ge),e(u,L),e(L,ye),e(L,R),e(R,Ee),e(L,be),c(t,W,h),c(t,E,h),e(E,we),e(E,B),e(B,_e),e(E,Te),J=!0},p:Qe,i(t){J||(Ve(w.$$.fragment,t),J=!0)},o(t){Xe(w.$$.fragment,t),J=!1},d(t){a(v),t&&a(U),t&&a(m),Ye(w),t&&a(H),t&&a(k),t&&a(K),t&&a(f),t&&a(j),t&&a(y),t&&a(q),t&&a(u),t&&a(W),t&&a(E)}}}const at={local:"bertology",title:"BERTology"};function rt(Re){return Ze(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class st extends Ke{constructor(v){super();je(this,v,rt,tt,qe,{})}}export{st as default,at as metadata};
