import{S as vr,i as yr,s as wr,e as o,k as c,w as Y,t as i,M as Pr,c as a,d as t,m as p,a as n,x as Z,h as s,b as l,G as r,g as h,y as ee,q as te,o as re,B as oe,v as gr}from"../chunks/vendor-hf-doc-builder.js";import{T as Er}from"../chunks/Tip-hf-doc-builder.js";import{I as le}from"../chunks/IconCopyLink-hf-doc-builder.js";function Tr(Ie){let m,b,_,u,w;return{c(){m=o("p"),b=i("for PyTorch >= 1.14.0. JIT-mode could benefit any models for prediction and evaluaion since dict input is supported in jit.trace"),_=c(),u=o("p"),w=i(`for PyTorch < 1.14.0. JIT-mode could benefit models whose forward parameter order matches the tuple input order in jit.trace, like question-answering model
In the case where the forward parameter order does not match the tuple input order in jit.trace, like text-classification models, jit.trace will fail and we are capturing this with the exception here to make it fallback. Logging is used to notify users.`)},l(d){m=a(d,"P",{});var v=n(m);b=s(v,"for PyTorch >= 1.14.0. JIT-mode could benefit any models for prediction and evaluaion since dict input is supported in jit.trace"),v.forEach(t),_=p(d),u=a(d,"P",{});var I=n(u);w=s(I,`for PyTorch < 1.14.0. JIT-mode could benefit models whose forward parameter order matches the tuple input order in jit.trace, like question-answering model
In the case where the forward parameter order does not match the tuple input order in jit.trace, like text-classification models, jit.trace will fail and we are capturing this with the exception here to make it fallback. Logging is used to notify users.`),I.forEach(t)},m(d,v){h(d,m,v),r(m,b),h(d,_,v),h(d,u,v),r(u,w)},d(d){d&&t(m),d&&t(_),d&&t(u)}}}function $r(Ie){let m,b,_,u,w,d,v,I,Ye,ke,ae,Ze,xe,P,k,fe,U,et,ne,he,tt,rt,je,y,ot,ce,at,nt,X,it,st,Ae,g,x,pe,B,lt,de,ft,Se,j,ht,G,ct,pt,qe,E,A,me,O,dt,ue,mt,Ce,S,ut,M,_t,vt,ze,T,q,_e,F,yt,ve,wt,Le,C,Pt,H,gt,Et,Ne,$,z,ye,R,Tt,we,$t,Je,L,Ue,D,bt,W,It,Xe,N,K,Pe,kt,xt,ie,jt,ge,At,St,Q,Ee,qt,Ct,J,zt,Te,Lt,Nt,$e,Jt,Be;return d=new le({}),U=new le({}),B=new le({}),O=new le({}),F=new le({}),R=new le({}),L=new Er({props:{warning:!0,$$slots:{default:[Tr]},$$scope:{ctx:Ie}}}),{c(){m=o("meta"),b=c(),_=o("h1"),u=o("a"),w=o("span"),Y(d.$$.fragment),v=c(),I=o("span"),Ye=i("Efficient Inference on CPU"),ke=c(),ae=o("p"),Ze=i("This guide focuses on inferencing large models efficiently on CPU."),xe=c(),P=o("h2"),k=o("a"),fe=o("span"),Y(U.$$.fragment),et=c(),ne=o("span"),he=o("code"),tt=i("BetterTransformer"),rt=i(" for faster inference"),je=c(),y=o("p"),ot=i("We have recently integrated "),ce=o("code"),at=i("BetterTransformer"),nt=i(" for faster inference on CPU for text, image and audio models. Check the documentation about this integration "),X=o("a"),it=i("here"),st=i(" for more details."),Ae=c(),g=o("h2"),x=o("a"),pe=o("span"),Y(B.$$.fragment),lt=c(),de=o("span"),ft=i("PyTorch JIT-mode (TorchScript)"),Se=i(`

TorchScript is a way to create serializable and optimizable models from PyTorch code. Any TorchScript program can be saved from a Python process and loaded in a process where there is no Python dependency.
Comparing to default eager mode, jit mode in PyTorch normally yields better performance for model inference from optimization methodologies like operator fusion.
`),j=o("p"),ht=i("For a gentle introduction to TorchScript, see the Introduction to "),G=o("a"),ct=i("PyTorch TorchScript tutorial"),pt=i("."),qe=c(),E=o("h3"),A=o("a"),me=o("span"),Y(O.$$.fragment),dt=c(),ue=o("span"),mt=i("IPEX Graph Optimization with JIT-mode"),Ce=i(`

Intel\xAE Extension for PyTorch provides further optimizations in jit mode for Transformers series models. It is highly recommended for users to take advantage of Intel\xAE Extension for PyTorch with jit mode. Some frequently used operator patterns from Transformers models are already supported in Intel\xAE Extension for PyTorch with jit mode fusions. Those fusion patterns like Multi-head-attention fusion, Concat Linear, Linear+Add, Linear+Gelu, Add+LayerNorm fusion and etc. are enabled and perform well. The benefit of the fusion is delivered to users in a transparent fashion. According to the analysis, ~70% of most popular NLP tasks in question-answering, text-classification, and token-classification can get performance benefits with these fusion patterns for both Float32 precision and BFloat16 Mixed precision.
`),S=o("p"),ut=i("Check more detailed information for "),M=o("a"),_t=i("IPEX Graph Optimization"),vt=i("."),ze=c(),T=o("h4"),q=o("a"),_e=o("span"),Y(F.$$.fragment),yt=c(),ve=o("span"),wt=i("IPEX installation:"),Le=c(),C=o("p"),Pt=i("IPEX release is following PyTorch, check the approaches for "),H=o("a"),gt=i("IPEX installation"),Et=i("."),Ne=c(),$=o("h3"),z=o("a"),ye=o("span"),Y(R.$$.fragment),Tt=c(),we=o("span"),$t=i("Usage of JIT-mode"),Je=i(`

To enable JIT-mode in Trainer for evaluaion or prediction, users should add \`jit_mode_eval\` in Trainer command arguments.
`),Y(L.$$.fragment),Ue=c(),D=o("p"),bt=i("Take an example of the use cases on "),W=o("a"),It=i("Transformers question-answering"),Xe=c(),N=o("ul"),K=o("li"),Pe=o("p"),kt=i("Inference using jit mode on CPU:"),xt=c(),ie=o("pre"),jt=i(`python run_qa.py \\
--model_name_or_path csarron/bert-base-uncased-squad-v1 \\
--dataset_name squad \\
--do_eval \\
--max_seq_length 384 \\
--doc_stride 128 \\
--output_dir /tmp/ \\
--no_cuda \\
`),ge=o("b"),At=i("--jit_mode_eval "),St=c(),Q=o("li"),Ee=o("p"),qt=i("Inference with IPEX using jit mode on CPU:"),Ct=c(),J=o("pre"),zt=i(`python run_qa.py \\
--model_name_or_path csarron/bert-base-uncased-squad-v1 \\
--dataset_name squad \\
--do_eval \\
--max_seq_length 384 \\
--doc_stride 128 \\
--output_dir /tmp/ \\
--no_cuda \\
`),Te=o("b"),Lt=i("--use_ipex \\"),Nt=i(`
`),$e=o("b"),Jt=i("--jit_mode_eval"),this.h()},l(e){const f=Pr('[data-svelte="svelte-1phssyn"]',document.head);m=a(f,"META",{name:!0,content:!0}),f.forEach(t),b=p(e),_=a(e,"H1",{class:!0});var V=n(_);u=a(V,"A",{id:!0,class:!0,href:!0});var Gt=n(u);w=a(Gt,"SPAN",{});var Ot=n(w);Z(d.$$.fragment,Ot),Ot.forEach(t),Gt.forEach(t),v=p(V),I=a(V,"SPAN",{});var Mt=n(I);Ye=s(Mt,"Efficient Inference on CPU"),Mt.forEach(t),V.forEach(t),ke=p(e),ae=a(e,"P",{});var Ft=n(ae);Ze=s(Ft,"This guide focuses on inferencing large models efficiently on CPU."),Ft.forEach(t),xe=p(e),P=a(e,"H2",{class:!0});var Ge=n(P);k=a(Ge,"A",{id:!0,class:!0,href:!0});var Ht=n(k);fe=a(Ht,"SPAN",{});var Rt=n(fe);Z(U.$$.fragment,Rt),Rt.forEach(t),Ht.forEach(t),et=p(Ge),ne=a(Ge,"SPAN",{});var Ut=n(ne);he=a(Ut,"CODE",{});var Dt=n(he);tt=s(Dt,"BetterTransformer"),Dt.forEach(t),rt=s(Ut," for faster inference"),Ut.forEach(t),Ge.forEach(t),je=p(e),y=a(e,"P",{});var se=n(y);ot=s(se,"We have recently integrated "),ce=a(se,"CODE",{});var Wt=n(ce);at=s(Wt,"BetterTransformer"),Wt.forEach(t),nt=s(se," for faster inference on CPU for text, image and audio models. Check the documentation about this integration "),X=a(se,"A",{href:!0,rel:!0});var Kt=n(X);it=s(Kt,"here"),Kt.forEach(t),st=s(se," for more details."),se.forEach(t),Ae=p(e),g=a(e,"H2",{class:!0});var Oe=n(g);x=a(Oe,"A",{id:!0,class:!0,href:!0});var Qt=n(x);pe=a(Qt,"SPAN",{});var Vt=n(pe);Z(B.$$.fragment,Vt),Vt.forEach(t),Qt.forEach(t),lt=p(Oe),de=a(Oe,"SPAN",{});var Yt=n(de);ft=s(Yt,"PyTorch JIT-mode (TorchScript)"),Yt.forEach(t),Oe.forEach(t),Se=s(e,`

TorchScript is a way to create serializable and optimizable models from PyTorch code. Any TorchScript program can be saved from a Python process and loaded in a process where there is no Python dependency.
Comparing to default eager mode, jit mode in PyTorch normally yields better performance for model inference from optimization methodologies like operator fusion.
`),j=a(e,"P",{});var Me=n(j);ht=s(Me,"For a gentle introduction to TorchScript, see the Introduction to "),G=a(Me,"A",{href:!0,rel:!0});var Zt=n(G);ct=s(Zt,"PyTorch TorchScript tutorial"),Zt.forEach(t),pt=s(Me,"."),Me.forEach(t),qe=p(e),E=a(e,"H3",{class:!0});var Fe=n(E);A=a(Fe,"A",{id:!0,class:!0,href:!0});var er=n(A);me=a(er,"SPAN",{});var tr=n(me);Z(O.$$.fragment,tr),tr.forEach(t),er.forEach(t),dt=p(Fe),ue=a(Fe,"SPAN",{});var rr=n(ue);mt=s(rr,"IPEX Graph Optimization with JIT-mode"),rr.forEach(t),Fe.forEach(t),Ce=s(e,`

Intel\xAE Extension for PyTorch provides further optimizations in jit mode for Transformers series models. It is highly recommended for users to take advantage of Intel\xAE Extension for PyTorch with jit mode. Some frequently used operator patterns from Transformers models are already supported in Intel\xAE Extension for PyTorch with jit mode fusions. Those fusion patterns like Multi-head-attention fusion, Concat Linear, Linear+Add, Linear+Gelu, Add+LayerNorm fusion and etc. are enabled and perform well. The benefit of the fusion is delivered to users in a transparent fashion. According to the analysis, ~70% of most popular NLP tasks in question-answering, text-classification, and token-classification can get performance benefits with these fusion patterns for both Float32 precision and BFloat16 Mixed precision.
`),S=a(e,"P",{});var He=n(S);ut=s(He,"Check more detailed information for "),M=a(He,"A",{href:!0,rel:!0});var or=n(M);_t=s(or,"IPEX Graph Optimization"),or.forEach(t),vt=s(He,"."),He.forEach(t),ze=p(e),T=a(e,"H4",{class:!0});var Re=n(T);q=a(Re,"A",{id:!0,class:!0,href:!0});var ar=n(q);_e=a(ar,"SPAN",{});var nr=n(_e);Z(F.$$.fragment,nr),nr.forEach(t),ar.forEach(t),yt=p(Re),ve=a(Re,"SPAN",{});var ir=n(ve);wt=s(ir,"IPEX installation:"),ir.forEach(t),Re.forEach(t),Le=p(e),C=a(e,"P",{});var De=n(C);Pt=s(De,"IPEX release is following PyTorch, check the approaches for "),H=a(De,"A",{href:!0,rel:!0});var sr=n(H);gt=s(sr,"IPEX installation"),sr.forEach(t),Et=s(De,"."),De.forEach(t),Ne=p(e),$=a(e,"H3",{class:!0});var We=n($);z=a(We,"A",{id:!0,class:!0,href:!0});var lr=n(z);ye=a(lr,"SPAN",{});var fr=n(ye);Z(R.$$.fragment,fr),fr.forEach(t),lr.forEach(t),Tt=p(We),we=a(We,"SPAN",{});var hr=n(we);$t=s(hr,"Usage of JIT-mode"),hr.forEach(t),We.forEach(t),Je=s(e,`

To enable JIT-mode in Trainer for evaluaion or prediction, users should add \`jit_mode_eval\` in Trainer command arguments.
`),Z(L.$$.fragment,e),Ue=p(e),D=a(e,"P",{});var Xt=n(D);bt=s(Xt,"Take an example of the use cases on "),W=a(Xt,"A",{href:!0,rel:!0});var cr=n(W);It=s(cr,"Transformers question-answering"),cr.forEach(t),Xt.forEach(t),Xe=p(e),N=a(e,"UL",{});var Ke=n(N);K=a(Ke,"LI",{});var Qe=n(K);Pe=a(Qe,"P",{});var pr=n(Pe);kt=s(pr,"Inference using jit mode on CPU:"),pr.forEach(t),xt=p(Qe),ie=a(Qe,"PRE",{});var Bt=n(ie);jt=s(Bt,`python run_qa.py \\
--model_name_or_path csarron/bert-base-uncased-squad-v1 \\
--dataset_name squad \\
--do_eval \\
--max_seq_length 384 \\
--doc_stride 128 \\
--output_dir /tmp/ \\
--no_cuda \\
`),ge=a(Bt,"B",{});var dr=n(ge);At=s(dr,"--jit_mode_eval "),dr.forEach(t),Bt.forEach(t),Qe.forEach(t),St=p(Ke),Q=a(Ke,"LI",{});var Ve=n(Q);Ee=a(Ve,"P",{});var mr=n(Ee);qt=s(mr,"Inference with IPEX using jit mode on CPU:"),mr.forEach(t),Ct=p(Ve),J=a(Ve,"PRE",{});var be=n(J);zt=s(be,`python run_qa.py \\
--model_name_or_path csarron/bert-base-uncased-squad-v1 \\
--dataset_name squad \\
--do_eval \\
--max_seq_length 384 \\
--doc_stride 128 \\
--output_dir /tmp/ \\
--no_cuda \\
`),Te=a(be,"B",{});var ur=n(Te);Lt=s(ur,"--use_ipex \\"),ur.forEach(t),Nt=s(be,`
`),$e=a(be,"B",{});var _r=n($e);Jt=s(_r,"--jit_mode_eval"),_r.forEach(t),be.forEach(t),Ve.forEach(t),Ke.forEach(t),this.h()},h(){l(m,"name","hf:doc:metadata"),l(m,"content",JSON.stringify(br)),l(u,"id","efficient-inference-on-cpu"),l(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(u,"href","#efficient-inference-on-cpu"),l(_,"class","relative group"),l(k,"id","bettertransformer-for-faster-inference"),l(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(k,"href","#bettertransformer-for-faster-inference"),l(P,"class","relative group"),l(X,"href","https://huggingface.co/docs/optimum/bettertransformer/overview"),l(X,"rel","nofollow"),l(x,"id","pytorch-jitmode-torchscript"),l(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(x,"href","#pytorch-jitmode-torchscript"),l(g,"class","relative group"),l(G,"href","https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html#tracing-modules"),l(G,"rel","nofollow"),l(A,"id","ipex-graph-optimization-with-jitmode"),l(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(A,"href","#ipex-graph-optimization-with-jitmode"),l(E,"class","relative group"),l(M,"href","https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/features/graph_optimization.html"),l(M,"rel","nofollow"),l(q,"id","ipex-installation"),l(q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(q,"href","#ipex-installation"),l(T,"class","relative group"),l(H,"href","https://intel.github.io/intel-extension-for-pytorch/"),l(H,"rel","nofollow"),l(z,"id","usage-of-jitmode"),l(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(z,"href","#usage-of-jitmode"),l($,"class","relative group"),l(W,"href","https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering"),l(W,"rel","nofollow")},m(e,f){r(document.head,m),h(e,b,f),h(e,_,f),r(_,u),r(u,w),ee(d,w,null),r(_,v),r(_,I),r(I,Ye),h(e,ke,f),h(e,ae,f),r(ae,Ze),h(e,xe,f),h(e,P,f),r(P,k),r(k,fe),ee(U,fe,null),r(P,et),r(P,ne),r(ne,he),r(he,tt),r(ne,rt),h(e,je,f),h(e,y,f),r(y,ot),r(y,ce),r(ce,at),r(y,nt),r(y,X),r(X,it),r(y,st),h(e,Ae,f),h(e,g,f),r(g,x),r(x,pe),ee(B,pe,null),r(g,lt),r(g,de),r(de,ft),h(e,Se,f),h(e,j,f),r(j,ht),r(j,G),r(G,ct),r(j,pt),h(e,qe,f),h(e,E,f),r(E,A),r(A,me),ee(O,me,null),r(E,dt),r(E,ue),r(ue,mt),h(e,Ce,f),h(e,S,f),r(S,ut),r(S,M),r(M,_t),r(S,vt),h(e,ze,f),h(e,T,f),r(T,q),r(q,_e),ee(F,_e,null),r(T,yt),r(T,ve),r(ve,wt),h(e,Le,f),h(e,C,f),r(C,Pt),r(C,H),r(H,gt),r(C,Et),h(e,Ne,f),h(e,$,f),r($,z),r(z,ye),ee(R,ye,null),r($,Tt),r($,we),r(we,$t),h(e,Je,f),ee(L,e,f),h(e,Ue,f),h(e,D,f),r(D,bt),r(D,W),r(W,It),h(e,Xe,f),h(e,N,f),r(N,K),r(K,Pe),r(Pe,kt),r(K,xt),r(K,ie),r(ie,jt),r(ie,ge),r(ge,At),r(N,St),r(N,Q),r(Q,Ee),r(Ee,qt),r(Q,Ct),r(Q,J),r(J,zt),r(J,Te),r(Te,Lt),r(J,Nt),r(J,$e),r($e,Jt),Be=!0},p(e,[f]){const V={};f&2&&(V.$$scope={dirty:f,ctx:e}),L.$set(V)},i(e){Be||(te(d.$$.fragment,e),te(U.$$.fragment,e),te(B.$$.fragment,e),te(O.$$.fragment,e),te(F.$$.fragment,e),te(R.$$.fragment,e),te(L.$$.fragment,e),Be=!0)},o(e){re(d.$$.fragment,e),re(U.$$.fragment,e),re(B.$$.fragment,e),re(O.$$.fragment,e),re(F.$$.fragment,e),re(R.$$.fragment,e),re(L.$$.fragment,e),Be=!1},d(e){t(m),e&&t(b),e&&t(_),oe(d),e&&t(ke),e&&t(ae),e&&t(xe),e&&t(P),oe(U),e&&t(je),e&&t(y),e&&t(Ae),e&&t(g),oe(B),e&&t(Se),e&&t(j),e&&t(qe),e&&t(E),oe(O),e&&t(Ce),e&&t(S),e&&t(ze),e&&t(T),oe(F),e&&t(Le),e&&t(C),e&&t(Ne),e&&t($),oe(R),e&&t(Je),oe(L,e),e&&t(Ue),e&&t(D),e&&t(Xe),e&&t(N)}}}const br={local:"efficient-inference-on-cpu",sections:[{local:"bettertransformer-for-faster-inference",title:"`BetterTransformer` for faster inference"},{local:"pytorch-jitmode-torchscript",sections:[{local:"ipex-graph-optimization-with-jitmode",sections:[{local:"ipex-installation",title:"IPEX installation:"}],title:"IPEX Graph Optimization with JIT-mode"},{local:"usage-of-jitmode",title:"Usage of JIT-mode"}],title:"PyTorch JIT-mode (TorchScript)"}],title:"Efficient Inference on CPU"};function Ir(Ie){return gr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ar extends vr{constructor(m){super();yr(this,m,Ir,$r,wr,{})}}export{Ar as default,br as metadata};
