import{S as Nt,i as Dt,s as Bt,e as i,k as c,w as K,t as n,M as xt,c as l,d as a,m as u,a as s,x as ee,h as o,b as p,G as t,g as d,y as te,L as At,q as ae,o as ne,B as oe,v as Vt}from"../../chunks/vendor-hf-doc-builder.js";import{I as Jt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as rt}from"../../chunks/CodeBlock-hf-doc-builder.js";function Xt(dt){let b,ie,_,g,V,Z,ke,X,Ue,le,N,Ee,se,M,Ze,D,Ce,$e,re,v,k,I,C,je,S,We,de,m,Ge,B,Te,Je,$,Ne,De,j,Q,Be,xe,z,Ae,Ve,pe,W,fe,U,Xe,G,R,Ie,Se,me,T,ce,E,Qe,P,ze,Re,ue,f,Pe,x,qe,Fe,q,Ye,Oe,F,Le,He,Y,Ke,et,O,tt,at,he,J,ye,y,nt,L,ot,it,H,lt,st,we;return Z=new Jt({}),C=new Jt({}),W=new rt({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBJTBBcGlwZWxpbmUlMjAlM0QlMjBTdGFibGVEaWZmdXNpb25QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTIycnVud2F5bWwlMkZzdGFibGUtZGlmZnVzaW9uLXYxLTUlMjIpJTBBcGlwZWxpbmUudW5ldC5jb25maWclNUIlMjJpbl9jaGFubmVscyUyMiU1RCUwQTQ=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipeline = StableDiffusionPipeline.from_pretrained(<span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>)
pipeline.unet.config[<span class="hljs-string">&quot;in_channels&quot;</span>]
<span class="hljs-number">4</span>`}}),T=new rt({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBJTBBcGlwZWxpbmUlMjAlM0QlMjBTdGFibGVEaWZmdXNpb25QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTIycnVud2F5bWwlMkZzdGFibGUtZGlmZnVzaW9uLWlucGFpbnRpbmclMjIpJTBBcGlwZWxpbmUudW5ldC5jb25maWclNUIlMjJpbl9jaGFubmVscyUyMiU1RCUwQTk=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipeline = StableDiffusionPipeline.from_pretrained(<span class="hljs-string">&quot;runwayml/stable-diffusion-inpainting&quot;</span>)
pipeline.unet.config[<span class="hljs-string">&quot;in_channels&quot;</span>]
<span class="hljs-number">9</span>`}}),J=new rt({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFVOZXQyRENvbmRpdGlvbk1vZGVsJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJydW53YXltbCUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyMiUwQXVuZXQlMjAlM0QlMjBVTmV0MkRDb25kaXRpb25Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwbW9kZWxfaWQlMkMlMjBzdWJmb2xkZXIlM0QlMjJ1bmV0JTIyJTJDJTIwaW5fY2hhbm5lbHMlM0Q5JTJDJTIwbG93X2NwdV9tZW1fdXNhZ2UlM0RGYWxzZSUyQyUyMGlnbm9yZV9taXNtYXRjaGVkX3NpemVzJTNEVHJ1ZSUwQSk=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> UNet2DConditionModel

model_id = <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>
unet = UNet2DConditionModel.from_pretrained(
    model_id, subfolder=<span class="hljs-string">&quot;unet&quot;</span>, in_channels=<span class="hljs-number">9</span>, low_cpu_mem_usage=<span class="hljs-literal">False</span>, ignore_mismatched_sizes=<span class="hljs-literal">True</span>
)`}}),{c(){b=i("meta"),ie=c(),_=i("h1"),g=i("a"),V=i("span"),K(Z.$$.fragment),ke=c(),X=i("span"),Ue=n("Adapt a model to a new task"),le=c(),N=i("p"),Ee=n("Many diffusion systems share the same components, allowing you to adapt a pretrained model for one task to an entirely different task."),se=c(),M=i("p"),Ze=n("This guide will show you how to adapt a pretrained text-to-image model for inpainting by initializing and modifying the architecture of a pretrained "),D=i("a"),Ce=n("UNet2DConditionModel"),$e=n("."),re=c(),v=i("h2"),k=i("a"),I=i("span"),K(C.$$.fragment),je=c(),S=i("span"),We=n("Configure UNet2DConditionModel parameters"),de=c(),m=i("p"),Ge=n("A "),B=i("a"),Te=n("UNet2DConditionModel"),Je=n(" by default accepts 4 channels in the "),$=i("a"),Ne=n("input sample"),De=n(". For example, load a pretrained text-to-image model like "),j=i("a"),Q=i("code"),Be=n("runwayml/stable-diffusion-v1-5"),xe=n(" and take a look at the number of "),z=i("code"),Ae=n("in_channels"),Ve=n(":"),pe=c(),K(W.$$.fragment),fe=c(),U=i("p"),Xe=n("Inpainting requires 9 channels in the input sample. You can check this value in a pretrained inpainting model like "),G=i("a"),R=i("code"),Ie=n("runwayml/stable-diffusion-inpainting"),Se=n(":"),me=c(),K(T.$$.fragment),ce=c(),E=i("p"),Qe=n("To adapt your text-to-image model for inpainting, you\u2019ll need to change the number of "),P=i("code"),ze=n("in_channels"),Re=n(" from 4 to 9."),ue=c(),f=i("p"),Pe=n("Initialize a "),x=i("a"),qe=n("UNet2DConditionModel"),Fe=n(" with the pretrained text-to-image model weights, and change "),q=i("code"),Ye=n("in_channels"),Oe=n(" to 9. Changing the number of "),F=i("code"),Le=n("in_channels"),He=n(" means you need to set "),Y=i("code"),Ke=n("ignore_mismatched_sizes=True"),et=n(" and "),O=i("code"),tt=n("low_cpu_mem_usage=False"),at=n(" to avoid a size mismatch error because the shape is different now."),he=c(),K(J.$$.fragment),ye=c(),y=i("p"),nt=n("The pretrained weights of the other components from the text-to-image model are initialized from their checkpoints, but the input channel weights ("),L=i("code"),ot=n("conv_in.weight"),it=n(") of the "),H=i("code"),lt=n("unet"),st=n(" are randomly initialized. It is important to finetune the model for inpainting because otherwise the model returns noise."),this.h()},l(e){const r=xt('[data-svelte="svelte-1phssyn"]',document.head);b=l(r,"META",{name:!0,content:!0}),r.forEach(a),ie=u(e),_=l(e,"H1",{class:!0});var be=s(_);g=l(be,"A",{id:!0,class:!0,href:!0});var pt=s(g);V=l(pt,"SPAN",{});var ft=s(V);ee(Z.$$.fragment,ft),ft.forEach(a),pt.forEach(a),ke=u(be),X=l(be,"SPAN",{});var mt=s(X);Ue=o(mt,"Adapt a model to a new task"),mt.forEach(a),be.forEach(a),le=u(e),N=l(e,"P",{});var ct=s(N);Ee=o(ct,"Many diffusion systems share the same components, allowing you to adapt a pretrained model for one task to an entirely different task."),ct.forEach(a),se=u(e),M=l(e,"P",{});var _e=s(M);Ze=o(_e,"This guide will show you how to adapt a pretrained text-to-image model for inpainting by initializing and modifying the architecture of a pretrained "),D=l(_e,"A",{href:!0});var ut=s(D);Ce=o(ut,"UNet2DConditionModel"),ut.forEach(a),$e=o(_e,"."),_e.forEach(a),re=u(e),v=l(e,"H2",{class:!0});var ve=s(v);k=l(ve,"A",{id:!0,class:!0,href:!0});var ht=s(k);I=l(ht,"SPAN",{});var yt=s(I);ee(C.$$.fragment,yt),yt.forEach(a),ht.forEach(a),je=u(ve),S=l(ve,"SPAN",{});var wt=s(S);We=o(wt,"Configure UNet2DConditionModel parameters"),wt.forEach(a),ve.forEach(a),de=u(e),m=l(e,"P",{});var w=s(m);Ge=o(w,"A "),B=l(w,"A",{href:!0});var bt=s(B);Te=o(bt,"UNet2DConditionModel"),bt.forEach(a),Je=o(w," by default accepts 4 channels in the "),$=l(w,"A",{href:!0,rel:!0});var _t=s($);Ne=o(_t,"input sample"),_t.forEach(a),De=o(w,". For example, load a pretrained text-to-image model like "),j=l(w,"A",{href:!0,rel:!0});var vt=s(j);Q=l(vt,"CODE",{});var gt=s(Q);Be=o(gt,"runwayml/stable-diffusion-v1-5"),gt.forEach(a),vt.forEach(a),xe=o(w," and take a look at the number of "),z=l(w,"CODE",{});var Mt=s(z);Ae=o(Mt,"in_channels"),Mt.forEach(a),Ve=o(w,":"),w.forEach(a),pe=u(e),ee(W.$$.fragment,e),fe=u(e),U=l(e,"P",{});var ge=s(U);Xe=o(ge,"Inpainting requires 9 channels in the input sample. You can check this value in a pretrained inpainting model like "),G=l(ge,"A",{href:!0,rel:!0});var kt=s(G);R=l(kt,"CODE",{});var Ut=s(R);Ie=o(Ut,"runwayml/stable-diffusion-inpainting"),Ut.forEach(a),kt.forEach(a),Se=o(ge,":"),ge.forEach(a),me=u(e),ee(T.$$.fragment,e),ce=u(e),E=l(e,"P",{});var Me=s(E);Qe=o(Me,"To adapt your text-to-image model for inpainting, you\u2019ll need to change the number of "),P=l(Me,"CODE",{});var Et=s(P);ze=o(Et,"in_channels"),Et.forEach(a),Re=o(Me," from 4 to 9."),Me.forEach(a),ue=u(e),f=l(e,"P",{});var h=s(f);Pe=o(h,"Initialize a "),x=l(h,"A",{href:!0});var Zt=s(x);qe=o(Zt,"UNet2DConditionModel"),Zt.forEach(a),Fe=o(h," with the pretrained text-to-image model weights, and change "),q=l(h,"CODE",{});var Ct=s(q);Ye=o(Ct,"in_channels"),Ct.forEach(a),Oe=o(h," to 9. Changing the number of "),F=l(h,"CODE",{});var $t=s(F);Le=o($t,"in_channels"),$t.forEach(a),He=o(h," means you need to set "),Y=l(h,"CODE",{});var jt=s(Y);Ke=o(jt,"ignore_mismatched_sizes=True"),jt.forEach(a),et=o(h," and "),O=l(h,"CODE",{});var Wt=s(O);tt=o(Wt,"low_cpu_mem_usage=False"),Wt.forEach(a),at=o(h," to avoid a size mismatch error because the shape is different now."),h.forEach(a),he=u(e),ee(J.$$.fragment,e),ye=u(e),y=l(e,"P",{});var A=s(y);nt=o(A,"The pretrained weights of the other components from the text-to-image model are initialized from their checkpoints, but the input channel weights ("),L=l(A,"CODE",{});var Gt=s(L);ot=o(Gt,"conv_in.weight"),Gt.forEach(a),it=o(A,") of the "),H=l(A,"CODE",{});var Tt=s(H);lt=o(Tt,"unet"),Tt.forEach(a),st=o(A," are randomly initialized. It is important to finetune the model for inpainting because otherwise the model returns noise."),A.forEach(a),this.h()},h(){p(b,"name","hf:doc:metadata"),p(b,"content",JSON.stringify(It)),p(g,"id","adapt-a-model-to-a-new-task"),p(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(g,"href","#adapt-a-model-to-a-new-task"),p(_,"class","relative group"),p(D,"href","/docs/diffusers/v0.19.0/en/api/models/unet2d-cond#diffusers.UNet2DConditionModel"),p(k,"id","configure-unet2dconditionmodel-parameters"),p(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(k,"href","#configure-unet2dconditionmodel-parameters"),p(v,"class","relative group"),p(B,"href","/docs/diffusers/v0.19.0/en/api/models/unet2d-cond#diffusers.UNet2DConditionModel"),p($,"href","https://huggingface.co/docs/diffusers/v0.16.0/en/api/models#diffusers.UNet2DConditionModel.in_channels"),p($,"rel","nofollow"),p(j,"href","https://huggingface.co/runwayml/stable-diffusion-v1-5"),p(j,"rel","nofollow"),p(G,"href","https://huggingface.co/runwayml/stable-diffusion-inpainting"),p(G,"rel","nofollow"),p(x,"href","/docs/diffusers/v0.19.0/en/api/models/unet2d-cond#diffusers.UNet2DConditionModel")},m(e,r){t(document.head,b),d(e,ie,r),d(e,_,r),t(_,g),t(g,V),te(Z,V,null),t(_,ke),t(_,X),t(X,Ue),d(e,le,r),d(e,N,r),t(N,Ee),d(e,se,r),d(e,M,r),t(M,Ze),t(M,D),t(D,Ce),t(M,$e),d(e,re,r),d(e,v,r),t(v,k),t(k,I),te(C,I,null),t(v,je),t(v,S),t(S,We),d(e,de,r),d(e,m,r),t(m,Ge),t(m,B),t(B,Te),t(m,Je),t(m,$),t($,Ne),t(m,De),t(m,j),t(j,Q),t(Q,Be),t(m,xe),t(m,z),t(z,Ae),t(m,Ve),d(e,pe,r),te(W,e,r),d(e,fe,r),d(e,U,r),t(U,Xe),t(U,G),t(G,R),t(R,Ie),t(U,Se),d(e,me,r),te(T,e,r),d(e,ce,r),d(e,E,r),t(E,Qe),t(E,P),t(P,ze),t(E,Re),d(e,ue,r),d(e,f,r),t(f,Pe),t(f,x),t(x,qe),t(f,Fe),t(f,q),t(q,Ye),t(f,Oe),t(f,F),t(F,Le),t(f,He),t(f,Y),t(Y,Ke),t(f,et),t(f,O),t(O,tt),t(f,at),d(e,he,r),te(J,e,r),d(e,ye,r),d(e,y,r),t(y,nt),t(y,L),t(L,ot),t(y,it),t(y,H),t(H,lt),t(y,st),we=!0},p:At,i(e){we||(ae(Z.$$.fragment,e),ae(C.$$.fragment,e),ae(W.$$.fragment,e),ae(T.$$.fragment,e),ae(J.$$.fragment,e),we=!0)},o(e){ne(Z.$$.fragment,e),ne(C.$$.fragment,e),ne(W.$$.fragment,e),ne(T.$$.fragment,e),ne(J.$$.fragment,e),we=!1},d(e){a(b),e&&a(ie),e&&a(_),oe(Z),e&&a(le),e&&a(N),e&&a(se),e&&a(M),e&&a(re),e&&a(v),oe(C),e&&a(de),e&&a(m),e&&a(pe),oe(W,e),e&&a(fe),e&&a(U),e&&a(me),oe(T,e),e&&a(ce),e&&a(E),e&&a(ue),e&&a(f),e&&a(he),oe(J,e),e&&a(ye),e&&a(y)}}}const It={local:"adapt-a-model-to-a-new-task",sections:[{local:"configure-unet2dconditionmodel-parameters",title:"Configure UNet2DConditionModel parameters"}],title:"Adapt a model to a new task"};function St(dt){return Vt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pt extends Nt{constructor(b){super();Dt(this,b,St,Xt,Bt,{})}}export{Pt as default,It as metadata};
