import{S as Ht,i as Jt,s as Ut,e as o,k as u,w as N,t as l,M as Kt,c as s,d as t,m as c,a as r,x as I,h as p,b as i,G as a,g as h,y as z,L as Vt,q as Q,o as G,B as H,v as Wt}from"../../chunks/vendor-hf-doc-builder.js";import{I as ce}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as Gt}from"../../chunks/CodeBlock-hf-doc-builder.js";function Yt(ft){let g,fe,v,$,B,D,Te,R,Me,de,w,y,X,S,Fe,Z,Oe,me,k,Ne,C,Ie,ze,ge,J,Qe,ve,U,ee,Ge,we,_,x,te,q,He,ae,Je,_e,f,Ue,oe,Ke,Ve,se,We,Ye,L,Be,Re,re,Xe,Ze,Pe,P,A,ne,j,et,ie,tt,be,K,at,$e,V,W,le,ot,st,ye,T,ke,b,E,pe,M,rt,he,nt,xe,d,it,F,lt,pt,ue,ht,ut,Ae,O,Ee;return D=new ce({}),S=new ce({}),q=new ce({}),j=new ce({}),T=new Gt({props:{code:`from transformers import AutoProcessor, Pix2StructForConditionalGeneration
import requests
from PIL import Image

model = Pix2StructForConditionalGeneration.from_pretrained("google/deplot")
processor = AutoProcessor.from_pretrained("google/deplot")
url = "https://raw.githubusercontent.com/vis-nlp/ChartQA/main/ChartQA%20Dataset/val/png/5090.png"
image = Image.open(requests.get(url, stream=True).raw)

inputs = processor(images=image, text="Generate underlying data table of the figure below:", return_tensors="pt")
predictions = model.generate(**inputs, max_new_tokens=512)
print(processor.decode(predictions[0], skip_special_tokens=True))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, Pix2StructForConditionalGeneration
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

model = Pix2StructForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
url = <span class="hljs-string">&quot;https://raw.githubusercontent.com/vis-nlp/ChartQA/main/ChartQA%20Dataset/val/png/5090.png&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

inputs = processor(images=image, text=<span class="hljs-string">&quot;Generate underlying data table of the figure below:&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
predictions = model.generate(**inputs, max_new_tokens=<span class="hljs-number">512</span>)
<span class="hljs-built_in">print</span>(processor.decode(predictions[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`}}),M=new ce({}),O=new Gt({props:{code:`from transformers.optimization import Adafactor, get_cosine_schedule_with_warmup

optimizer = Adafactor(self.parameters(), scale_parameter=False, relative_step=False, lr=0.01, weight_decay=1e-05)
scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=1000, num_training_steps=40000)`,highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, get_cosine_schedule_with_warmup

optimizer = Adafactor(self.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">0.01</span>, weight_decay=<span class="hljs-number">1e-05</span>)
scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=<span class="hljs-number">1000</span>, num_training_steps=<span class="hljs-number">40000</span>)`}}),{c(){g=o("meta"),fe=u(),v=o("h1"),$=o("a"),B=o("span"),N(D.$$.fragment),Te=u(),R=o("span"),Me=l("DePlot"),de=u(),w=o("h2"),y=o("a"),X=o("span"),N(S.$$.fragment),Fe=u(),Z=o("span"),Oe=l("Overview"),me=u(),k=o("p"),Ne=l("DePlot was proposed in the paper "),C=o("a"),Ie=l("DePlot: One-shot visual language reasoning by plot-to-table translation"),ze=l(" from Fangyu Liu, Julian Martin Eisenschlos, Francesco Piccinno, Syrine Krichene, Chenxi Pang, Kenton Lee, Mandar Joshi, Wenhu Chen, Nigel Collier, Yasemin Altun."),ge=u(),J=o("p"),Qe=l("The abstract of the paper states the following:"),ve=u(),U=o("p"),ee=o("em"),Ge=l("Visual language such as charts and plots is ubiquitous in the human world. Comprehending plots and charts requires strong reasoning skills. Prior state-of-the-art (SOTA) models require at least tens of thousands of training examples and their reasoning capabilities are still much limited, especially on complex human-written queries. This paper presents the first one-shot solution to visual language reasoning. We decompose the challenge of visual language reasoning into two steps: (1) plot-to-text translation, and (2) reasoning over the translated text. The key in this method is a modality conversion module, named as DePlot, which translates the image of a plot or chart to a linearized table. The output of DePlot can then be directly used to prompt a pretrained large language model (LLM), exploiting the few-shot reasoning capabilities of LLMs. To obtain DePlot, we standardize the plot-to-table task by establishing unified task formats and metrics, and train DePlot end-to-end on this task. DePlot can then be used off-the-shelf together with LLMs in a plug-and-play fashion. Compared with a SOTA model finetuned on more than >28k data points, DePlot+LLM with just one-shot prompting achieves a 24.0% improvement over finetuned SOTA on human-written queries from the task of chart QA."),we=u(),_=o("h2"),x=o("a"),te=o("span"),N(q.$$.fragment),He=u(),ae=o("span"),Je=l("Model description"),_e=u(),f=o("p"),Ue=l("DePlot is a model that is trained using "),oe=o("code"),Ke=l("Pix2Struct"),Ve=l(" architecture. You can find more information about "),se=o("code"),We=l("Pix2Struct"),Ye=l(" in the "),L=o("a"),Be=l("Pix2Struct documentation"),Re=l(`.
DePlot is a Visual Question Answering subset of `),re=o("code"),Xe=l("Pix2Struct"),Ze=l(" architecture. It renders the input question on the image and predicts the answer."),Pe=u(),P=o("h2"),A=o("a"),ne=o("span"),N(j.$$.fragment),et=u(),ie=o("span"),tt=l("Usage"),be=u(),K=o("p"),at=l("Currently one checkpoint is available for DePlot:"),$e=u(),V=o("ul"),W=o("li"),le=o("code"),ot=l("google/deplot"),st=l(": DePlot fine-tuned on ChartQA dataset"),ye=u(),N(T.$$.fragment),ke=u(),b=o("h2"),E=o("a"),pe=o("span"),N(M.$$.fragment),rt=u(),he=o("span"),nt=l("Fine-tuning"),xe=u(),d=o("p"),it=l("To fine-tune DePlot, refer to the pix2struct "),F=o("a"),lt=l("fine-tuning notebook"),pt=l(". For "),ue=o("code"),ht=l("Pix2Struct"),ut=l(" models, we have found out that fine-tuning the model with Adafactor and cosine learning rate scheduler leads to faster convergence:"),Ae=u(),N(O.$$.fragment),this.h()},l(e){const n=Kt('[data-svelte="svelte-1phssyn"]',document.head);g=s(n,"META",{name:!0,content:!0}),n.forEach(t),fe=c(e),v=s(e,"H1",{class:!0});var De=r(v);$=s(De,"A",{id:!0,class:!0,href:!0});var dt=r($);B=s(dt,"SPAN",{});var mt=r(B);I(D.$$.fragment,mt),mt.forEach(t),dt.forEach(t),Te=c(De),R=s(De,"SPAN",{});var gt=r(R);Me=p(gt,"DePlot"),gt.forEach(t),De.forEach(t),de=c(e),w=s(e,"H2",{class:!0});var Se=r(w);y=s(Se,"A",{id:!0,class:!0,href:!0});var vt=r(y);X=s(vt,"SPAN",{});var wt=r(X);I(S.$$.fragment,wt),wt.forEach(t),vt.forEach(t),Fe=c(Se),Z=s(Se,"SPAN",{});var _t=r(Z);Oe=p(_t,"Overview"),_t.forEach(t),Se.forEach(t),me=c(e),k=s(e,"P",{});var Ce=r(k);Ne=p(Ce,"DePlot was proposed in the paper "),C=s(Ce,"A",{href:!0,rel:!0});var Pt=r(C);Ie=p(Pt,"DePlot: One-shot visual language reasoning by plot-to-table translation"),Pt.forEach(t),ze=p(Ce," from Fangyu Liu, Julian Martin Eisenschlos, Francesco Piccinno, Syrine Krichene, Chenxi Pang, Kenton Lee, Mandar Joshi, Wenhu Chen, Nigel Collier, Yasemin Altun."),Ce.forEach(t),ge=c(e),J=s(e,"P",{});var bt=r(J);Qe=p(bt,"The abstract of the paper states the following:"),bt.forEach(t),ve=c(e),U=s(e,"P",{});var $t=r(U);ee=s($t,"EM",{});var yt=r(ee);Ge=p(yt,"Visual language such as charts and plots is ubiquitous in the human world. Comprehending plots and charts requires strong reasoning skills. Prior state-of-the-art (SOTA) models require at least tens of thousands of training examples and their reasoning capabilities are still much limited, especially on complex human-written queries. This paper presents the first one-shot solution to visual language reasoning. We decompose the challenge of visual language reasoning into two steps: (1) plot-to-text translation, and (2) reasoning over the translated text. The key in this method is a modality conversion module, named as DePlot, which translates the image of a plot or chart to a linearized table. The output of DePlot can then be directly used to prompt a pretrained large language model (LLM), exploiting the few-shot reasoning capabilities of LLMs. To obtain DePlot, we standardize the plot-to-table task by establishing unified task formats and metrics, and train DePlot end-to-end on this task. DePlot can then be used off-the-shelf together with LLMs in a plug-and-play fashion. Compared with a SOTA model finetuned on more than >28k data points, DePlot+LLM with just one-shot prompting achieves a 24.0% improvement over finetuned SOTA on human-written queries from the task of chart QA."),yt.forEach(t),$t.forEach(t),we=c(e),_=s(e,"H2",{class:!0});var qe=r(_);x=s(qe,"A",{id:!0,class:!0,href:!0});var kt=r(x);te=s(kt,"SPAN",{});var xt=r(te);I(q.$$.fragment,xt),xt.forEach(t),kt.forEach(t),He=c(qe),ae=s(qe,"SPAN",{});var At=r(ae);Je=p(At,"Model description"),At.forEach(t),qe.forEach(t),_e=c(e),f=s(e,"P",{});var m=r(f);Ue=p(m,"DePlot is a model that is trained using "),oe=s(m,"CODE",{});var Et=r(oe);Ke=p(Et,"Pix2Struct"),Et.forEach(t),Ve=p(m," architecture. You can find more information about "),se=s(m,"CODE",{});var Dt=r(se);We=p(Dt,"Pix2Struct"),Dt.forEach(t),Ye=p(m," in the "),L=s(m,"A",{href:!0,rel:!0});var St=r(L);Be=p(St,"Pix2Struct documentation"),St.forEach(t),Re=p(m,`.
DePlot is a Visual Question Answering subset of `),re=s(m,"CODE",{});var Ct=r(re);Xe=p(Ct,"Pix2Struct"),Ct.forEach(t),Ze=p(m," architecture. It renders the input question on the image and predicts the answer."),m.forEach(t),Pe=c(e),P=s(e,"H2",{class:!0});var Le=r(P);A=s(Le,"A",{id:!0,class:!0,href:!0});var qt=r(A);ne=s(qt,"SPAN",{});var Lt=r(ne);I(j.$$.fragment,Lt),Lt.forEach(t),qt.forEach(t),et=c(Le),ie=s(Le,"SPAN",{});var jt=r(ie);tt=p(jt,"Usage"),jt.forEach(t),Le.forEach(t),be=c(e),K=s(e,"P",{});var Tt=r(K);at=p(Tt,"Currently one checkpoint is available for DePlot:"),Tt.forEach(t),$e=c(e),V=s(e,"UL",{});var Mt=r(V);W=s(Mt,"LI",{});var ct=r(W);le=s(ct,"CODE",{});var Ft=r(le);ot=p(Ft,"google/deplot"),Ft.forEach(t),st=p(ct,": DePlot fine-tuned on ChartQA dataset"),ct.forEach(t),Mt.forEach(t),ye=c(e),I(T.$$.fragment,e),ke=c(e),b=s(e,"H2",{class:!0});var je=r(b);E=s(je,"A",{id:!0,class:!0,href:!0});var Ot=r(E);pe=s(Ot,"SPAN",{});var Nt=r(pe);I(M.$$.fragment,Nt),Nt.forEach(t),Ot.forEach(t),rt=c(je),he=s(je,"SPAN",{});var It=r(he);nt=p(It,"Fine-tuning"),It.forEach(t),je.forEach(t),xe=c(e),d=s(e,"P",{});var Y=r(d);it=p(Y,"To fine-tune DePlot, refer to the pix2struct "),F=s(Y,"A",{href:!0,rel:!0});var zt=r(F);lt=p(zt,"fine-tuning notebook"),zt.forEach(t),pt=p(Y,". For "),ue=s(Y,"CODE",{});var Qt=r(ue);ht=p(Qt,"Pix2Struct"),Qt.forEach(t),ut=p(Y," models, we have found out that fine-tuning the model with Adafactor and cosine learning rate scheduler leads to faster convergence:"),Y.forEach(t),Ae=c(e),I(O.$$.fragment,e),this.h()},h(){i(g,"name","hf:doc:metadata"),i(g,"content",JSON.stringify(Bt)),i($,"id","deplot"),i($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i($,"href","#deplot"),i(v,"class","relative group"),i(y,"id","overview"),i(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(y,"href","#overview"),i(w,"class","relative group"),i(C,"href","https://arxiv.org/abs/2212.10505"),i(C,"rel","nofollow"),i(x,"id","model-description"),i(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(x,"href","#model-description"),i(_,"class","relative group"),i(L,"href","https://huggingface.co/docs/transformers/main/en/model_doc/pix2struct"),i(L,"rel","nofollow"),i(A,"id","usage"),i(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(A,"href","#usage"),i(P,"class","relative group"),i(E,"id","finetuning"),i(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(E,"href","#finetuning"),i(b,"class","relative group"),i(F,"href","https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_pix2struct.ipynb"),i(F,"rel","nofollow")},m(e,n){a(document.head,g),h(e,fe,n),h(e,v,n),a(v,$),a($,B),z(D,B,null),a(v,Te),a(v,R),a(R,Me),h(e,de,n),h(e,w,n),a(w,y),a(y,X),z(S,X,null),a(w,Fe),a(w,Z),a(Z,Oe),h(e,me,n),h(e,k,n),a(k,Ne),a(k,C),a(C,Ie),a(k,ze),h(e,ge,n),h(e,J,n),a(J,Qe),h(e,ve,n),h(e,U,n),a(U,ee),a(ee,Ge),h(e,we,n),h(e,_,n),a(_,x),a(x,te),z(q,te,null),a(_,He),a(_,ae),a(ae,Je),h(e,_e,n),h(e,f,n),a(f,Ue),a(f,oe),a(oe,Ke),a(f,Ve),a(f,se),a(se,We),a(f,Ye),a(f,L),a(L,Be),a(f,Re),a(f,re),a(re,Xe),a(f,Ze),h(e,Pe,n),h(e,P,n),a(P,A),a(A,ne),z(j,ne,null),a(P,et),a(P,ie),a(ie,tt),h(e,be,n),h(e,K,n),a(K,at),h(e,$e,n),h(e,V,n),a(V,W),a(W,le),a(le,ot),a(W,st),h(e,ye,n),z(T,e,n),h(e,ke,n),h(e,b,n),a(b,E),a(E,pe),z(M,pe,null),a(b,rt),a(b,he),a(he,nt),h(e,xe,n),h(e,d,n),a(d,it),a(d,F),a(F,lt),a(d,pt),a(d,ue),a(ue,ht),a(d,ut),h(e,Ae,n),z(O,e,n),Ee=!0},p:Vt,i(e){Ee||(Q(D.$$.fragment,e),Q(S.$$.fragment,e),Q(q.$$.fragment,e),Q(j.$$.fragment,e),Q(T.$$.fragment,e),Q(M.$$.fragment,e),Q(O.$$.fragment,e),Ee=!0)},o(e){G(D.$$.fragment,e),G(S.$$.fragment,e),G(q.$$.fragment,e),G(j.$$.fragment,e),G(T.$$.fragment,e),G(M.$$.fragment,e),G(O.$$.fragment,e),Ee=!1},d(e){t(g),e&&t(fe),e&&t(v),H(D),e&&t(de),e&&t(w),H(S),e&&t(me),e&&t(k),e&&t(ge),e&&t(J),e&&t(ve),e&&t(U),e&&t(we),e&&t(_),H(q),e&&t(_e),e&&t(f),e&&t(Pe),e&&t(P),H(j),e&&t(be),e&&t(K),e&&t($e),e&&t(V),e&&t(ye),H(T,e),e&&t(ke),e&&t(b),H(M),e&&t(xe),e&&t(d),e&&t(Ae),H(O,e)}}}const Bt={local:"deplot",sections:[{local:"overview",title:"Overview "},{local:"model-description",title:"Model description"},{local:"usage",title:"Usage"},{local:"finetuning",title:"Fine-tuning"}],title:"DePlot"};function Rt(ft){return Wt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ta extends Ht{constructor(g){super();Jt(this,g,Rt,Yt,Ut,{})}}export{ta as default,Bt as metadata};
