import{S as ei,i as ii,s as ti,e as n,k as c,w as oe,t as s,M as ni,c as a,d as i,m,a as r,x as re,h as l,b as f,G as t,g as d,y as se,q as le,o as fe,B as de,v as ai}from"../../../chunks/vendor-hf-doc-builder.js";import{T as oi}from"../../../chunks/Tip-hf-doc-builder.js";import{D as ri}from"../../../chunks/Docstring-hf-doc-builder.js";import{I as Ze}from"../../../chunks/IconCopyLink-hf-doc-builder.js";function si(W){let p,_;return{c(){p=n("p"),_=s("\u{1F9EA} This is an experimental pipeline for reinforcement learning!")},l(u){p=a(u,"P",{});var h=r(p);_=l(h,"\u{1F9EA} This is an experimental pipeline for reinforcement learning!"),h.forEach(i)},m(u,h){d(u,p,h),t(p,_)},d(u){u&&i(p)}}}function li(W){let p,_,u,h,U,E,pe,I,ue,Y,w,F,y,he,k,ce,me,H,N,ge,O,D,q,ve,be,K,g,_e,M,we,ye,A,$e,Pe,L,xe,Ee,Q,$,ke,S,De,Me,X,b,P,B,T,Ae,J,Le,Z,v,V,Se,C,Te,Ve,j,je,R,Ge,Ne,ee;return E=new Ze({}),w=new oi({props:{warning:!0,$$slots:{default:[si]},$$scope:{ctx:W}}}),T=new Ze({}),V=new ri({props:{name:"class diffusers.experimental.ValueGuidedRLPipeline",anchor:"diffusers.experimental.ValueGuidedRLPipeline",parameters:[{name:"value_function",val:": UNet1DModel"},{name:"unet",val:": UNet1DModel"},{name:"scheduler",val:": DDPMScheduler"},{name:"env",val:""}],parametersDescription:[{anchor:"diffusers.experimental.ValueGuidedRLPipeline.value_function",description:`<strong>value_function</strong> (<a href="/docs/diffusers/v0.21.0/en/api/models/unet#diffusers.UNet1DModel">UNet1DModel</a>) &#x2014;
A specialized UNet for fine-tuning trajectories base on reward.`,name:"value_function"},{anchor:"diffusers.experimental.ValueGuidedRLPipeline.unet",description:`<strong>unet</strong> (<a href="/docs/diffusers/v0.21.0/en/api/models/unet#diffusers.UNet1DModel">UNet1DModel</a>) &#x2014;
UNet architecture to denoise the encoded trajectories.`,name:"unet"},{anchor:"diffusers.experimental.ValueGuidedRLPipeline.scheduler",description:`<strong>scheduler</strong> (<a href="/docs/diffusers/v0.21.0/en/api/schedulers/overview#diffusers.SchedulerMixin">SchedulerMixin</a>) &#x2014;
A scheduler to be used in combination with <code>unet</code> to denoise the encoded trajectories. Default for this
application is <a href="/docs/diffusers/v0.21.0/en/api/schedulers/ddpm#diffusers.DDPMScheduler">DDPMScheduler</a>.`,name:"scheduler"},{anchor:"diffusers.experimental.ValueGuidedRLPipeline.env",description:`<strong>env</strong> () &#x2014;
An environment following the OpenAI gym API to act in. For now only Hopper has pretrained models.`,name:"env"}],source:"https://github.com/huggingface/diffusers/blob/v0.21.0/src/diffusers/experimental/rl/value_guided_sampling.py#L25"}}),{c(){p=n("meta"),_=c(),u=n("h1"),h=n("a"),U=n("span"),oe(E.$$.fragment),pe=c(),I=n("span"),ue=s("Value-guided planning"),Y=c(),oe(w.$$.fragment),F=c(),y=n("p"),he=s("This pipeline is based on the "),k=n("a"),ce=s("Planning with Diffusion for Flexible Behavior Synthesis"),me=s(" paper by Michael Janner, Yilun Du, Joshua B. Tenenbaum, Sergey Levine."),H=c(),N=n("p"),ge=s("The abstract from the paper is:"),O=c(),D=n("p"),q=n("em"),ve=s("Model-based reinforcement learning methods often use learning only for the purpose of estimating an approximate dynamics model, offloading the rest of the decision-making work to classical trajectory optimizers. While conceptually simple, this combination has a number of empirical shortcomings, suggesting that learned models may not be well-suited to standard trajectory optimization. In this paper, we consider what it would look like to fold as much of the trajectory optimization pipeline as possible into the modeling problem, such that sampling from the model and planning with it become nearly identical. The core of our technical approach lies in a diffusion probabilistic model that plans by iteratively denoising trajectories. We show how classifier-guided sampling and image inpainting can be reinterpreted as coherent planning strategies, explore the unusual and useful properties of diffusion-based planning methods, and demonstrate the effectiveness of our framework in control settings that emphasize long-horizon decision-making and test-time flexibility"),be=s("."),K=c(),g=n("p"),_e=s("You can find additional information about the model on the "),M=n("a"),we=s("project page"),ye=s(", the "),A=n("a"),$e=s("original codebase"),Pe=s(", or try it out in a demo "),L=n("a"),xe=s("notebook"),Ee=s("."),Q=c(),$=n("p"),ke=s("The script to run the model is available "),S=n("a"),De=s("here"),Me=s("."),X=c(),b=n("h2"),P=n("a"),B=n("span"),oe(T.$$.fragment),Ae=c(),J=n("span"),Le=s("ValueGuidedRLPipeline"),Z=c(),v=n("div"),oe(V.$$.fragment),Se=c(),C=n("p"),Te=s("Pipeline for value-guided sampling from a diffusion model trained to predict sequences of states."),Ve=c(),j=n("p"),je=s("This model inherits from "),R=n("a"),Ge=s("DiffusionPipeline"),Ne=s(`. Check the superclass documentation for the generic methods
implemented for all pipelines (downloading, saving, running on a particular device, etc.).`),this.h()},l(e){const o=ni('[data-svelte="svelte-1phssyn"]',document.head);p=a(o,"META",{name:!0,content:!0}),o.forEach(i),_=m(e),u=a(e,"H1",{class:!0});var G=r(u);h=a(G,"A",{id:!0,class:!0,href:!0});var ze=r(h);U=a(ze,"SPAN",{});var Ue=r(U);re(E.$$.fragment,Ue),Ue.forEach(i),ze.forEach(i),pe=m(G),I=a(G,"SPAN",{});var Ie=r(I);ue=l(Ie,"Value-guided planning"),Ie.forEach(i),G.forEach(i),Y=m(e),re(w.$$.fragment,e),F=m(e),y=a(e,"P",{});var ie=r(y);he=l(ie,"This pipeline is based on the "),k=a(ie,"A",{href:!0,rel:!0});var qe=r(k);ce=l(qe,"Planning with Diffusion for Flexible Behavior Synthesis"),qe.forEach(i),me=l(ie," paper by Michael Janner, Yilun Du, Joshua B. Tenenbaum, Sergey Levine."),ie.forEach(i),H=m(e),N=a(e,"P",{});var Be=r(N);ge=l(Be,"The abstract from the paper is:"),Be.forEach(i),O=m(e),D=a(e,"P",{});var Re=r(D);q=a(Re,"EM",{});var Je=r(q);ve=l(Je,"Model-based reinforcement learning methods often use learning only for the purpose of estimating an approximate dynamics model, offloading the rest of the decision-making work to classical trajectory optimizers. While conceptually simple, this combination has a number of empirical shortcomings, suggesting that learned models may not be well-suited to standard trajectory optimization. In this paper, we consider what it would look like to fold as much of the trajectory optimization pipeline as possible into the modeling problem, such that sampling from the model and planning with it become nearly identical. The core of our technical approach lies in a diffusion probabilistic model that plans by iteratively denoising trajectories. We show how classifier-guided sampling and image inpainting can be reinterpreted as coherent planning strategies, explore the unusual and useful properties of diffusion-based planning methods, and demonstrate the effectiveness of our framework in control settings that emphasize long-horizon decision-making and test-time flexibility"),Je.forEach(i),be=l(Re,"."),Re.forEach(i),K=m(e),g=a(e,"P",{});var x=r(g);_e=l(x,"You can find additional information about the model on the "),M=a(x,"A",{href:!0,rel:!0});var Ce=r(M);we=l(Ce,"project page"),Ce.forEach(i),ye=l(x,", the "),A=a(x,"A",{href:!0,rel:!0});var We=r(A);$e=l(We,"original codebase"),We.forEach(i),Pe=l(x,", or try it out in a demo "),L=a(x,"A",{href:!0,rel:!0});var Ye=r(L);xe=l(Ye,"notebook"),Ye.forEach(i),Ee=l(x,"."),x.forEach(i),Q=m(e),$=a(e,"P",{});var te=r($);ke=l(te,"The script to run the model is available "),S=a(te,"A",{href:!0,rel:!0});var Fe=r(S);De=l(Fe,"here"),Fe.forEach(i),Me=l(te,"."),te.forEach(i),X=m(e),b=a(e,"H2",{class:!0});var ne=r(b);P=a(ne,"A",{id:!0,class:!0,href:!0});var He=r(P);B=a(He,"SPAN",{});var Oe=r(B);re(T.$$.fragment,Oe),Oe.forEach(i),He.forEach(i),Ae=m(ne),J=a(ne,"SPAN",{});var Ke=r(J);Le=l(Ke,"ValueGuidedRLPipeline"),Ke.forEach(i),ne.forEach(i),Z=m(e),v=a(e,"DIV",{class:!0});var z=r(v);re(V.$$.fragment,z),Se=m(z),C=a(z,"P",{});var Qe=r(C);Te=l(Qe,"Pipeline for value-guided sampling from a diffusion model trained to predict sequences of states."),Qe.forEach(i),Ve=m(z),j=a(z,"P",{});var ae=r(j);je=l(ae,"This model inherits from "),R=a(ae,"A",{href:!0});var Xe=r(R);Ge=l(Xe,"DiffusionPipeline"),Xe.forEach(i),Ne=l(ae,`. Check the superclass documentation for the generic methods
implemented for all pipelines (downloading, saving, running on a particular device, etc.).`),ae.forEach(i),z.forEach(i),this.h()},h(){f(p,"name","hf:doc:metadata"),f(p,"content",JSON.stringify(fi)),f(h,"id","valueguided-planning"),f(h,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(h,"href","#valueguided-planning"),f(u,"class","relative group"),f(k,"href","https://huggingface.co/papers/2205.09991"),f(k,"rel","nofollow"),f(M,"href","https://diffusion-planning.github.io/"),f(M,"rel","nofollow"),f(A,"href","https://github.com/jannerm/diffuser"),f(A,"rel","nofollow"),f(L,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/reinforcement_learning_with_diffusers.ipynb"),f(L,"rel","nofollow"),f(S,"href","https://github.com/huggingface/diffusers/tree/main/examples/reinforcement_learning"),f(S,"rel","nofollow"),f(P,"id","diffusers.experimental.ValueGuidedRLPipeline"),f(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(P,"href","#diffusers.experimental.ValueGuidedRLPipeline"),f(b,"class","relative group"),f(R,"href","/docs/diffusers/v0.21.0/en/api/pipelines/overview#diffusers.DiffusionPipeline"),f(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,o){t(document.head,p),d(e,_,o),d(e,u,o),t(u,h),t(h,U),se(E,U,null),t(u,pe),t(u,I),t(I,ue),d(e,Y,o),se(w,e,o),d(e,F,o),d(e,y,o),t(y,he),t(y,k),t(k,ce),t(y,me),d(e,H,o),d(e,N,o),t(N,ge),d(e,O,o),d(e,D,o),t(D,q),t(q,ve),t(D,be),d(e,K,o),d(e,g,o),t(g,_e),t(g,M),t(M,we),t(g,ye),t(g,A),t(A,$e),t(g,Pe),t(g,L),t(L,xe),t(g,Ee),d(e,Q,o),d(e,$,o),t($,ke),t($,S),t(S,De),t($,Me),d(e,X,o),d(e,b,o),t(b,P),t(P,B),se(T,B,null),t(b,Ae),t(b,J),t(J,Le),d(e,Z,o),d(e,v,o),se(V,v,null),t(v,Se),t(v,C),t(C,Te),t(v,Ve),t(v,j),t(j,je),t(j,R),t(R,Ge),t(j,Ne),ee=!0},p(e,[o]){const G={};o&2&&(G.$$scope={dirty:o,ctx:e}),w.$set(G)},i(e){ee||(le(E.$$.fragment,e),le(w.$$.fragment,e),le(T.$$.fragment,e),le(V.$$.fragment,e),ee=!0)},o(e){fe(E.$$.fragment,e),fe(w.$$.fragment,e),fe(T.$$.fragment,e),fe(V.$$.fragment,e),ee=!1},d(e){i(p),e&&i(_),e&&i(u),de(E),e&&i(Y),de(w,e),e&&i(F),e&&i(y),e&&i(H),e&&i(N),e&&i(O),e&&i(D),e&&i(K),e&&i(g),e&&i(Q),e&&i($),e&&i(X),e&&i(b),de(T),e&&i(Z),e&&i(v),de(V)}}}const fi={local:"valueguided-planning",sections:[{local:"diffusers.experimental.ValueGuidedRLPipeline",title:"ValueGuidedRLPipeline"}],title:"Value-guided planning"};function di(W){return ai(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class mi extends ei{constructor(p){super();ii(this,p,di,li,ti,{})}}export{mi as default,fi as metadata};
