import{S as Yt,i as It,s as Nt,e as s,k as c,w as B,t as i,M as Ft,c as n,d as t,m as u,a as o,x as C,h,b as r,G as a,g as p,y as G,L as zt,q as X,o as V,B as Y,v as Lt}from"../../chunks/vendor-hf-doc-builder.js";import{I as ue}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as Vt}from"../../chunks/CodeBlock-hf-doc-builder.js";function qt(dt){let g,de,w,_,H,$,De,Q,We,fe,v,k,O,j,Se,K,Ze,me,J,Be,x,Ce,Ge,ge,I,Xe,we,N,ee,Ve,ve,y,E,te,R,Ye,ae,Ie,ye,d,Ne,se,Fe,ze,ne,Le,qe,U,He,Qe,oe,Oe,Ke,be,b,P,le,A,et,re,tt,Me,F,at,_e,z,L,ie,st,nt,ke,D,Je,M,T,he,W,ot,pe,lt,Ee,f,rt,S,it,ht,ce,pt,ct,Pe,Z,Te;return $=new ue({}),j=new ue({}),R=new ue({}),A=new ue({}),D=new Vt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMkMlMjBQaXgyU3RydWN0Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEElMEFtb2RlbCUyMCUzRCUyMFBpeDJTdHJ1Y3RGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmRlcGxvdCUyMiklMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUlMkZkZXBsb3QlMjIpJTBBdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZyYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tJTJGdmlzLW5scCUyRkNoYXJ0UUElMkZtYWluJTJGQ2hhcnRRQSUyNTIwRGF0YXNldCUyRnZhbCUyRnBuZyUyRjUwOTAucG5nJTIyJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlcyUzRGltYWdlJTJDJTIwdGV4dCUzRCUyMkdlbmVyYXRlJTIwdW5kZXJseWluZyUyMGRhdGElMjB0YWJsZSUyMG9mJTIwdGhlJTIwZmlndXJlJTIwYmVsb3clM0ElMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQXByZWRpY3Rpb25zJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDUxMiklMEFwcmludChwcm9jZXNzb3IuZGVjb2RlKHByZWRpY3Rpb25zJTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, Pix2StructForConditionalGeneration
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

model = Pix2StructForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
url = <span class="hljs-string">&quot;https://raw.githubusercontent.com/vis-nlp/ChartQA/main/ChartQA%20Dataset/val/png/5090.png&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

inputs = processor(images=image, text=<span class="hljs-string">&quot;Generate underlying data table of the figure below:&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
predictions = model.generate(**inputs, max_new_tokens=<span class="hljs-number">512</span>)
<span class="hljs-built_in">print</span>(processor.decode(predictions[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`}}),W=new ue({}),Z=new Vt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5vcHRpbWl6YXRpb24lMjBpbXBvcnQlMjBBZGFmYWN0b3IlMkMlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwJTBBJTBBb3B0aW1pemVyJTIwJTNEJTIwQWRhZmFjdG9yKHNlbGYucGFyYW1ldGVycygpJTJDJTIwc2NhbGVfcGFyYW1ldGVyJTNERmFsc2UlMkMlMjByZWxhdGl2ZV9zdGVwJTNERmFsc2UlMkMlMjBsciUzRDAuMDElMkMlMjB3ZWlnaHRfZGVjYXklM0QxZS0wNSklMEFzY2hlZHVsZXIlMjAlM0QlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwKG9wdGltaXplciUyQyUyMG51bV93YXJtdXBfc3RlcHMlM0QxMDAwJTJDJTIwbnVtX3RyYWluaW5nX3N0ZXBzJTNENDAwMDAp",highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, get_cosine_schedule_with_warmup

optimizer = Adafactor(self.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">0.01</span>, weight_decay=<span class="hljs-number">1e-05</span>)
scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=<span class="hljs-number">1000</span>, num_training_steps=<span class="hljs-number">40000</span>)`}}),{c(){g=s("meta"),de=c(),w=s("h1"),_=s("a"),H=s("span"),B($.$$.fragment),De=c(),Q=s("span"),We=i("DePlot"),fe=c(),v=s("h2"),k=s("a"),O=s("span"),B(j.$$.fragment),Se=c(),K=s("span"),Ze=i("Overview"),me=c(),J=s("p"),Be=i("DePlot was proposed in the paper "),x=s("a"),Ce=i("DePlot: One-shot visual language reasoning by plot-to-table translation"),Ge=i(" from Fangyu Liu, Julian Martin Eisenschlos, Francesco Piccinno, Syrine Krichene, Chenxi Pang, Kenton Lee, Mandar Joshi, Wenhu Chen, Nigel Collier, Yasemin Altun."),ge=c(),I=s("p"),Xe=i("The abstract of the paper states the following:"),we=c(),N=s("p"),ee=s("em"),Ve=i("Visual language such as charts and plots is ubiquitous in the human world. Comprehending plots and charts requires strong reasoning skills. Prior state-of-the-art (SOTA) models require at least tens of thousands of training examples and their reasoning capabilities are still much limited, especially on complex human-written queries. This paper presents the first one-shot solution to visual language reasoning. We decompose the challenge of visual language reasoning into two steps: (1) plot-to-text translation, and (2) reasoning over the translated text. The key in this method is a modality conversion module, named as DePlot, which translates the image of a plot or chart to a linearized table. The output of DePlot can then be directly used to prompt a pretrained large language model (LLM), exploiting the few-shot reasoning capabilities of LLMs. To obtain DePlot, we standardize the plot-to-table task by establishing unified task formats and metrics, and train DePlot end-to-end on this task. DePlot can then be used off-the-shelf together with LLMs in a plug-and-play fashion. Compared with a SOTA model finetuned on more than >28k data points, DePlot+LLM with just one-shot prompting achieves a 24.0% improvement over finetuned SOTA on human-written queries from the task of chart QA."),ve=c(),y=s("h2"),E=s("a"),te=s("span"),B(R.$$.fragment),Ye=c(),ae=s("span"),Ie=i("Model description"),ye=c(),d=s("p"),Ne=i("DePlot is a model that is trained using "),se=s("code"),Fe=i("Pix2Struct"),ze=i(" architecture. You can find more information about "),ne=s("code"),Le=i("Pix2Struct"),qe=i(" in the "),U=s("a"),He=i("Pix2Struct documentation"),Qe=i(`.
DePlot is a Visual Question Answering subset of `),oe=s("code"),Oe=i("Pix2Struct"),Ke=i(" architecture. It renders the input question on the image and predicts the answer."),be=c(),b=s("h2"),P=s("a"),le=s("span"),B(A.$$.fragment),et=c(),re=s("span"),tt=i("Usage"),Me=c(),F=s("p"),at=i("Currently one checkpoint is available for DePlot:"),_e=c(),z=s("ul"),L=s("li"),ie=s("code"),st=i("google/deplot"),nt=i(": DePlot fine-tuned on ChartQA dataset"),ke=c(),B(D.$$.fragment),Je=c(),M=s("h2"),T=s("a"),he=s("span"),B(W.$$.fragment),ot=c(),pe=s("span"),lt=i("Fine-tuning"),Ee=c(),f=s("p"),rt=i("To fine-tune DePlot, refer to the pix2struct "),S=s("a"),it=i("fine-tuning notebook"),ht=i(". For "),ce=s("code"),pt=i("Pix2Struct"),ct=i(" models, we have found out that fine-tuning the model with Adafactor and cosine learning rate scheduler leads to faster convergence:"),Pe=c(),B(Z.$$.fragment),this.h()},l(e){const l=Ft('[data-svelte="svelte-1phssyn"]',document.head);g=n(l,"META",{name:!0,content:!0}),l.forEach(t),de=u(e),w=n(e,"H1",{class:!0});var $e=o(w);_=n($e,"A",{id:!0,class:!0,href:!0});var ft=o(_);H=n(ft,"SPAN",{});var mt=o(H);C($.$$.fragment,mt),mt.forEach(t),ft.forEach(t),De=u($e),Q=n($e,"SPAN",{});var gt=o(Q);We=h(gt,"DePlot"),gt.forEach(t),$e.forEach(t),fe=u(e),v=n(e,"H2",{class:!0});var je=o(v);k=n(je,"A",{id:!0,class:!0,href:!0});var wt=o(k);O=n(wt,"SPAN",{});var vt=o(O);C(j.$$.fragment,vt),vt.forEach(t),wt.forEach(t),Se=u(je),K=n(je,"SPAN",{});var yt=o(K);Ze=h(yt,"Overview"),yt.forEach(t),je.forEach(t),me=u(e),J=n(e,"P",{});var xe=o(J);Be=h(xe,"DePlot was proposed in the paper "),x=n(xe,"A",{href:!0,rel:!0});var bt=o(x);Ce=h(bt,"DePlot: One-shot visual language reasoning by plot-to-table translation"),bt.forEach(t),Ge=h(xe," from Fangyu Liu, Julian Martin Eisenschlos, Francesco Piccinno, Syrine Krichene, Chenxi Pang, Kenton Lee, Mandar Joshi, Wenhu Chen, Nigel Collier, Yasemin Altun."),xe.forEach(t),ge=u(e),I=n(e,"P",{});var Mt=o(I);Xe=h(Mt,"The abstract of the paper states the following:"),Mt.forEach(t),we=u(e),N=n(e,"P",{});var _t=o(N);ee=n(_t,"EM",{});var kt=o(ee);Ve=h(kt,"Visual language such as charts and plots is ubiquitous in the human world. Comprehending plots and charts requires strong reasoning skills. Prior state-of-the-art (SOTA) models require at least tens of thousands of training examples and their reasoning capabilities are still much limited, especially on complex human-written queries. This paper presents the first one-shot solution to visual language reasoning. We decompose the challenge of visual language reasoning into two steps: (1) plot-to-text translation, and (2) reasoning over the translated text. The key in this method is a modality conversion module, named as DePlot, which translates the image of a plot or chart to a linearized table. The output of DePlot can then be directly used to prompt a pretrained large language model (LLM), exploiting the few-shot reasoning capabilities of LLMs. To obtain DePlot, we standardize the plot-to-table task by establishing unified task formats and metrics, and train DePlot end-to-end on this task. DePlot can then be used off-the-shelf together with LLMs in a plug-and-play fashion. Compared with a SOTA model finetuned on more than >28k data points, DePlot+LLM with just one-shot prompting achieves a 24.0% improvement over finetuned SOTA on human-written queries from the task of chart QA."),kt.forEach(t),_t.forEach(t),ve=u(e),y=n(e,"H2",{class:!0});var Re=o(y);E=n(Re,"A",{id:!0,class:!0,href:!0});var Jt=o(E);te=n(Jt,"SPAN",{});var Et=o(te);C(R.$$.fragment,Et),Et.forEach(t),Jt.forEach(t),Ye=u(Re),ae=n(Re,"SPAN",{});var Pt=o(ae);Ie=h(Pt,"Model description"),Pt.forEach(t),Re.forEach(t),ye=u(e),d=n(e,"P",{});var m=o(d);Ne=h(m,"DePlot is a model that is trained using "),se=n(m,"CODE",{});var Tt=o(se);Fe=h(Tt,"Pix2Struct"),Tt.forEach(t),ze=h(m," architecture. You can find more information about "),ne=n(m,"CODE",{});var $t=o(ne);Le=h($t,"Pix2Struct"),$t.forEach(t),qe=h(m," in the "),U=n(m,"A",{href:!0,rel:!0});var jt=o(U);He=h(jt,"Pix2Struct documentation"),jt.forEach(t),Qe=h(m,`.
DePlot is a Visual Question Answering subset of `),oe=n(m,"CODE",{});var xt=o(oe);Oe=h(xt,"Pix2Struct"),xt.forEach(t),Ke=h(m," architecture. It renders the input question on the image and predicts the answer."),m.forEach(t),be=u(e),b=n(e,"H2",{class:!0});var Ue=o(b);P=n(Ue,"A",{id:!0,class:!0,href:!0});var Rt=o(P);le=n(Rt,"SPAN",{});var Ut=o(le);C(A.$$.fragment,Ut),Ut.forEach(t),Rt.forEach(t),et=u(Ue),re=n(Ue,"SPAN",{});var At=o(re);tt=h(At,"Usage"),At.forEach(t),Ue.forEach(t),Me=u(e),F=n(e,"P",{});var Dt=o(F);at=h(Dt,"Currently one checkpoint is available for DePlot:"),Dt.forEach(t),_e=u(e),z=n(e,"UL",{});var Wt=o(z);L=n(Wt,"LI",{});var ut=o(L);ie=n(ut,"CODE",{});var St=o(ie);st=h(St,"google/deplot"),St.forEach(t),nt=h(ut,": DePlot fine-tuned on ChartQA dataset"),ut.forEach(t),Wt.forEach(t),ke=u(e),C(D.$$.fragment,e),Je=u(e),M=n(e,"H2",{class:!0});var Ae=o(M);T=n(Ae,"A",{id:!0,class:!0,href:!0});var Zt=o(T);he=n(Zt,"SPAN",{});var Bt=o(he);C(W.$$.fragment,Bt),Bt.forEach(t),Zt.forEach(t),ot=u(Ae),pe=n(Ae,"SPAN",{});var Ct=o(pe);lt=h(Ct,"Fine-tuning"),Ct.forEach(t),Ae.forEach(t),Ee=u(e),f=n(e,"P",{});var q=o(f);rt=h(q,"To fine-tune DePlot, refer to the pix2struct "),S=n(q,"A",{href:!0,rel:!0});var Gt=o(S);it=h(Gt,"fine-tuning notebook"),Gt.forEach(t),ht=h(q,". For "),ce=n(q,"CODE",{});var Xt=o(ce);pt=h(Xt,"Pix2Struct"),Xt.forEach(t),ct=h(q," models, we have found out that fine-tuning the model with Adafactor and cosine learning rate scheduler leads to faster convergence:"),q.forEach(t),Pe=u(e),C(Z.$$.fragment,e),this.h()},h(){r(g,"name","hf:doc:metadata"),r(g,"content",JSON.stringify(Ht)),r(_,"id","deplot"),r(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(_,"href","#deplot"),r(w,"class","relative group"),r(k,"id","overview"),r(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(k,"href","#overview"),r(v,"class","relative group"),r(x,"href","https://arxiv.org/abs/2212.10505"),r(x,"rel","nofollow"),r(E,"id","model-description"),r(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(E,"href","#model-description"),r(y,"class","relative group"),r(U,"href","https://huggingface.co/docs/transformers/main/en/model_doc/pix2struct"),r(U,"rel","nofollow"),r(P,"id","usage"),r(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(P,"href","#usage"),r(b,"class","relative group"),r(T,"id","finetuning"),r(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(T,"href","#finetuning"),r(M,"class","relative group"),r(S,"href","https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_pix2struct.ipynb"),r(S,"rel","nofollow")},m(e,l){a(document.head,g),p(e,de,l),p(e,w,l),a(w,_),a(_,H),G($,H,null),a(w,De),a(w,Q),a(Q,We),p(e,fe,l),p(e,v,l),a(v,k),a(k,O),G(j,O,null),a(v,Se),a(v,K),a(K,Ze),p(e,me,l),p(e,J,l),a(J,Be),a(J,x),a(x,Ce),a(J,Ge),p(e,ge,l),p(e,I,l),a(I,Xe),p(e,we,l),p(e,N,l),a(N,ee),a(ee,Ve),p(e,ve,l),p(e,y,l),a(y,E),a(E,te),G(R,te,null),a(y,Ye),a(y,ae),a(ae,Ie),p(e,ye,l),p(e,d,l),a(d,Ne),a(d,se),a(se,Fe),a(d,ze),a(d,ne),a(ne,Le),a(d,qe),a(d,U),a(U,He),a(d,Qe),a(d,oe),a(oe,Oe),a(d,Ke),p(e,be,l),p(e,b,l),a(b,P),a(P,le),G(A,le,null),a(b,et),a(b,re),a(re,tt),p(e,Me,l),p(e,F,l),a(F,at),p(e,_e,l),p(e,z,l),a(z,L),a(L,ie),a(ie,st),a(L,nt),p(e,ke,l),G(D,e,l),p(e,Je,l),p(e,M,l),a(M,T),a(T,he),G(W,he,null),a(M,ot),a(M,pe),a(pe,lt),p(e,Ee,l),p(e,f,l),a(f,rt),a(f,S),a(S,it),a(f,ht),a(f,ce),a(ce,pt),a(f,ct),p(e,Pe,l),G(Z,e,l),Te=!0},p:zt,i(e){Te||(X($.$$.fragment,e),X(j.$$.fragment,e),X(R.$$.fragment,e),X(A.$$.fragment,e),X(D.$$.fragment,e),X(W.$$.fragment,e),X(Z.$$.fragment,e),Te=!0)},o(e){V($.$$.fragment,e),V(j.$$.fragment,e),V(R.$$.fragment,e),V(A.$$.fragment,e),V(D.$$.fragment,e),V(W.$$.fragment,e),V(Z.$$.fragment,e),Te=!1},d(e){t(g),e&&t(de),e&&t(w),Y($),e&&t(fe),e&&t(v),Y(j),e&&t(me),e&&t(J),e&&t(ge),e&&t(I),e&&t(we),e&&t(N),e&&t(ve),e&&t(y),Y(R),e&&t(ye),e&&t(d),e&&t(be),e&&t(b),Y(A),e&&t(Me),e&&t(F),e&&t(_e),e&&t(z),e&&t(ke),Y(D,e),e&&t(Je),e&&t(M),Y(W),e&&t(Ee),e&&t(f),e&&t(Pe),Y(Z,e)}}}const Ht={local:"deplot",sections:[{local:"overview",title:"Overview "},{local:"model-description",title:"Model description"},{local:"usage",title:"Usage"},{local:"finetuning",title:"Fine-tuning"}],title:"DePlot"};function Qt(dt){return Lt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ta extends Yt{constructor(g){super();It(this,g,Qt,qt,Nt,{})}}export{ta as default,Ht as metadata};
