import{S as nf,i as rf,s as pf,e as s,k as m,w as y,t as n,M as mf,c as a,d as l,m as f,a as o,x as w,h as r,b as c,G as t,g as p,y as M,q as J,o as T,B as b,v as ff}from"../../chunks/vendor-hf-doc-builder.js";import{T as uo}from"../../chunks/Tip-hf-doc-builder.js";import{I as R}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as B}from"../../chunks/CodeBlock-hf-doc-builder.js";function cf(W){let d,u,h,U,v,_;return{c(){d=s("p"),u=n("It is strongly discouraged to make use of "),h=s("a"),U=s("code"),v=n("torch.autocast"),_=n(` in any of the pipelines as it can lead to black images and is always slower than using pure
float16 precision.`),this.h()},l(S){d=a(S,"P",{});var Z=o(d);u=r(Z,"It is strongly discouraged to make use of "),h=a(Z,"A",{href:!0,rel:!0});var ut=o(h);U=a(ut,"CODE",{});var Ne=o(U);v=r(Ne,"torch.autocast"),Ne.forEach(l),ut.forEach(l),_=r(Z,` in any of the pipelines as it can lead to black images and is always slower than using pure
float16 precision.`),Z.forEach(l),this.h()},h(){c(h,"href","https://pytorch.org/docs/stable/amp.html#torch.autocast"),c(h,"rel","nofollow")},m(S,Z){p(S,d,Z),t(d,u),t(d,h),t(h,U),t(U,v),t(d,_)},d(S){S&&l(d)}}}function df(W){let d,u,h,U;return{c(){d=n("Consider using "),u=s("a"),h=n("model offloading"),U=n(" as another point in the optimization space: it will be much faster, but memory savings won't be as large."),this.h()},l(v){d=r(v,"Consider using "),u=a(v,"A",{href:!0});var _=o(u);h=r(_,"model offloading"),_.forEach(l),U=r(v," as another point in the optimization space: it will be much faster, but memory savings won't be as large."),this.h()},h(){c(u,"href","#model_offloading")},m(v,_){p(v,d,_),p(v,u,_),t(u,h),p(v,U,_)},d(v){v&&l(d),v&&l(u),v&&l(U)}}}function hf(W){let d;return{c(){d=n("This feature requires `accelerate` version 0.17.0 or larger.")},l(u){d=r(u,"This feature requires `accelerate` version 0.17.0 or larger.")},m(u,h){p(u,d,h)},d(u){u&&l(d)}}}function uf(W){let d,u;return{c(){d=s("p"),u=n("If you have PyTorch 2.0 installed, you shouldn\u2019t use xFormers!")},l(h){d=a(h,"P",{});var U=o(d);u=r(U,"If you have PyTorch 2.0 installed, you shouldn\u2019t use xFormers!"),U.forEach(l)},m(h,U){p(h,d,U),t(d,u)},d(h){h&&l(d)}}}function yf(W){let d,u,h,U,v,_,S,Z,ut,Ne,C,yo,rl,wo,Mo,Re,Jo,To,yt,bo,Uo,$s,wt,vo,As,ce,il,X,Ns,_o,pl,jo,Eo,ml,Io,go,I,Y,fl,Zo,Co,cl,Bo,Go,dl,Wo,ko,Q,hl,Vo,$o,ul,Ao,No,yl,Ro,So,z,wl,Xo,Yo,Ml,Qo,zo,Jl,xo,Fo,x,Tl,Ho,Do,bl,Po,qo,Ul,Lo,Ko,F,vl,Oo,en,_l,tn,ln,jl,sn,Rs,Mt,an,Ss,H,de,El,Se,on,Il,nn,Xs,he,rn,Xe,pn,mn,Ys,Ye,Qs,D,ue,gl,Qe,fn,Zl,cn,zs,k,dn,Cl,hn,un,Bl,yn,wn,xs,ze,Fs,ye,Hs,P,we,Gl,xe,Mn,Wl,Jn,Ds,Jt,Tn,Ps,Me,bn,Tt,Un,vn,qs,Je,_n,bt,jn,En,Ls,Fe,Ks,Ut,In,Os,q,Te,kl,He,gn,Vl,Zn,ea,vt,Cn,ta,be,Bn,_t,Gn,Wn,la,Ue,kn,jt,Vn,$n,sa,De,aa,Et,An,oa,It,na,L,ve,$l,Pe,Nn,Al,Rn,ra,gt,Sn,ia,_e,Xn,Zt,Yn,Qn,pa,qe,ma,Ct,zn,fa,je,xn,Nl,Fn,Hn,ca,Ee,da,Bt,Dn,ha,Le,ua,g,Rl,Pn,qn,Sl,Ln,Kn,Xl,On,er,Ke,tr,lr,ya,K,Yl,sr,ar,Ql,or,nr,wa,Gt,Ma,O,Ie,zl,Oe,rr,xl,ir,Ja,et,Wt,pr,mr,Ta,V,fr,Fl,cr,dr,Hl,hr,ur,ba,kt,yr,Ua,ge,wr,Dl,Mr,Jr,va,tt,_a,Vt,Tr,ja,lt,Ea,Ze,Ia,G,Pl,br,Ur,ql,vr,_r,st,jr,Er,ga,ee,Ce,Ll,at,Ir,Kl,gr,Za,$t,Zr,Ca,At,Cr,Ba,ot,Ga,te,Be,Ol,nt,Br,es,Gr,Wa,Ge,Wr,ts,kr,Vr,ka,Nt,$r,Va,rt,$a,We,Ar,ls,Nr,Rr,Aa,it,Na,le,ke,ss,pt,Sr,as,Xr,Ra,$,Yr,mt,Qr,zr,ft,xr,Fr,Sa,Rt,Hr,Xa,Ve,os,se,ns,Dr,Pr,rs,qr,Lr,is,Kr,Or,j,ae,ps,ei,ti,ms,li,si,fs,ai,oi,oe,cs,ni,ri,ds,ii,pi,hs,mi,fi,ne,us,ci,di,ys,hi,ui,ws,yi,wi,re,Ms,Mi,Ji,Js,Ti,bi,Ts,Ui,vi,ie,bs,_i,ji,Us,Ei,Ii,vs,gi,Zi,pe,_s,Ci,Bi,js,Gi,Wi,Es,ki,Vi,me,Is,$i,Ai,gs,Ni,Ri,Zs,Si,Ya,St,Xi,Qa,$e,za,A,Cs,Yi,Qi,Bs,zi,xi,Xt,Yt,Fi,Hi,xa,ct,Fa;return _=new R({}),Se=new R({}),Ye=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEF0b3JjaC5iYWNrZW5kcy5jdWRhLm1hdG11bC5hbGxvd190ZjMyJTIwJTNEJTIwVHJ1ZQ==",highlighted:`<span class="hljs-keyword">import</span> torch

torch.backends.cuda.matmul.allow_tf32 = <span class="hljs-literal">True</span>`}}),Qe=new R({}),ze=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEFwaXBlJTIwJTNEJTIwcGlwZS50byglMjJjdWRhJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMmElMjBwaG90byUyMG9mJTIwYW4lMjBhc3Ryb25hdXQlMjByaWRpbmclMjBhJTIwaG9yc2UlMjBvbiUyMG1hcnMlMjIlMEFpbWFnZSUyMCUzRCUyMHBpcGUocHJvbXB0KS5pbWFnZXMlNUIwJTVE",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline

pipe = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)
pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),ye=new uo({props:{warning:!0,$$slots:{default:[cf]},$$scope:{ctx:W}}}),xe=new R({}),Fe=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEFwaXBlJTIwJTNEJTIwcGlwZS50byglMjJjdWRhJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMmElMjBwaG90byUyMG9mJTIwYW4lMjBhc3Ryb25hdXQlMjByaWRpbmclMjBhJTIwaG9yc2UlMjBvbiUyMG1hcnMlMjIlMEFwaXBlLmVuYWJsZV92YWVfc2xpY2luZygpJTBBaW1hZ2VzJTIwJTNEJTIwcGlwZSglNUJwcm9tcHQlNUQlMjAqJTIwMzIpLmltYWdlcw==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)
pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_vae_slicing()
images = pipe([prompt] * <span class="hljs-number">32</span>).images`}}),He=new R({}),De=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMkMlMjBVbmlQQ011bHRpc3RlcFNjaGVkdWxlciUwQSUwQXBpcGUlMjAlM0QlMjBTdGFibGVEaWZmdXNpb25QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIycnVud2F5bWwlMkZzdGFibGUtZGlmZnVzaW9uLXYxLTUlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEElMjAlMjAlMjAlMjB1c2Vfc2FmZXRlbnNvcnMlM0RUcnVlJTJDJTBBKSUwQXBpcGUuc2NoZWR1bGVyJTIwJTNEJTIwVW5pUENNdWx0aXN0ZXBTY2hlZHVsZXIuZnJvbV9jb25maWcocGlwZS5zY2hlZHVsZXIuY29uZmlnKSUwQXBpcGUlMjAlM0QlMjBwaXBlLnRvKCUyMmN1ZGElMjIpJTBBcHJvbXB0JTIwJTNEJTIwJTIyYSUyMGJlYXV0aWZ1bCUyMGxhbmRzY2FwZSUyMHBob3RvZ3JhcGglMjIlMEFwaXBlLmVuYWJsZV92YWVfdGlsaW5nKCklMEFwaXBlLmVuYWJsZV94Zm9ybWVyc19tZW1vcnlfZWZmaWNpZW50X2F0dGVudGlvbigpJTBBJTBBaW1hZ2UlMjAlM0QlMjBwaXBlKCU1QnByb21wdCU1RCUyQyUyMHdpZHRoJTNEMzg0MCUyQyUyMGhlaWdodCUzRDIyMjQlMkMlMjBudW1faW5mZXJlbmNlX3N0ZXBzJTNEMjApLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline, UniPCMultistepScheduler

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)
prompt = <span class="hljs-string">&quot;a beautiful landscape photograph&quot;</span>
pipe.enable_vae_tiling()
pipe.enable_xformers_memory_efficient_attention()

image = pipe([prompt], width=<span class="hljs-number">3840</span>, height=<span class="hljs-number">2224</span>, num_inference_steps=<span class="hljs-number">20</span>).images[<span class="hljs-number">0</span>]`}}),Pe=new R({}),qe=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjJhJTIwcGhvdG8lMjBvZiUyMGFuJTIwYXN0cm9uYXV0JTIwcmlkaW5nJTIwYSUyMGhvcnNlJTIwb24lMjBtYXJzJTIyJTBBcGlwZS5lbmFibGVfc2VxdWVudGlhbF9jcHVfb2ZmbG9hZCgpJTBBaW1hZ2UlMjAlM0QlMjBwaXBlKHByb21wdCkuaW1hZ2VzJTVCMCU1RA==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_sequential_cpu_offload()
image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),Ee=new uo({props:{$$slots:{default:[df]},$$scope:{ctx:W}}}),Le=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjJhJTIwcGhvdG8lMjBvZiUyMGFuJTIwYXN0cm9uYXV0JTIwcmlkaW5nJTIwYSUyMGhvcnNlJTIwb24lMjBtYXJzJTIyJTBBcGlwZS5lbmFibGVfc2VxdWVudGlhbF9jcHVfb2ZmbG9hZCgpJTBBJTBBaW1hZ2UlMjAlM0QlMjBwaXBlKHByb21wdCkuaW1hZ2VzJTVCMCU1RA==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_sequential_cpu_offload()

image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),Oe=new R({}),tt=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTIwJTIwJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjJhJTIwcGhvdG8lMjBvZiUyMGFuJTIwYXN0cm9uYXV0JTIwcmlkaW5nJTIwYSUyMGhvcnNlJTIwb24lMjBtYXJzJTIyJTBBcGlwZS5lbmFibGVfbW9kZWxfY3B1X29mZmxvYWQoKSUwQWltYWdlJTIwJTNEJTIwcGlwZShwcm9tcHQpLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,  
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_model_cpu_offload()
image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),lt=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjJhJTIwcGhvdG8lMjBvZiUyMGFuJTIwYXN0cm9uYXV0JTIwcmlkaW5nJTIwYSUyMGhvcnNlJTIwb24lMjBtYXJzJTIyJTBBcGlwZS5lbmFibGVfbW9kZWxfY3B1X29mZmxvYWQoKSUwQSUwQWltYWdlJTIwJTNEJTIwcGlwZShwcm9tcHQpLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_model_cpu_offload()

image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),Ze=new uo({props:{$$slots:{default:[hf]},$$scope:{ctx:W}}}),at=new R({}),ot=new B({props:{code:"cHJpbnQocGlwZS51bmV0LmNvbnZfb3V0LnN0YXRlX2RpY3QoKSU1QiUyMndlaWdodCUyMiU1RC5zdHJpZGUoKSklMjAlMjAlMjMlMjAoMjg4MCUyQyUyMDklMkMlMjAzJTJDJTIwMSklMEFwaXBlLnVuZXQudG8obWVtb3J5X2Zvcm1hdCUzRHRvcmNoLmNoYW5uZWxzX2xhc3QpJTIwJTIwJTIzJTIwaW4tcGxhY2UlMjBvcGVyYXRpb24lMEFwcmludCglMEElMjAlMjAlMjAlMjBwaXBlLnVuZXQuY29udl9vdXQuc3RhdGVfZGljdCgpJTVCJTIyd2VpZ2h0JTIyJTVELnN0cmlkZSgpJTBBKSUyMCUyMCUyMyUyMCgyODgwJTJDJTIwMSUyQyUyMDk2MCUyQyUyMDMyMCklMjBoYXZpbmclMjBhJTIwc3RyaWRlJTIwb2YlMjAxJTIwZm9yJTIwdGhlJTIwMm5kJTIwZGltZW5zaW9uJTIwcHJvdmVzJTIwdGhhdCUyMGl0JTIwd29ya3M=",highlighted:`<span class="hljs-built_in">print</span>(pipe.unet.conv_out.state_dict()[<span class="hljs-string">&quot;weight&quot;</span>].stride())  <span class="hljs-comment"># (2880, 9, 3, 1)</span>
pipe.unet.to(memory_format=torch.channels_last)  <span class="hljs-comment"># in-place operation</span>
<span class="hljs-built_in">print</span>(
    pipe.unet.conv_out.state_dict()[<span class="hljs-string">&quot;weight&quot;</span>].stride()
)  <span class="hljs-comment"># (2880, 1, 960, 320) having a stride of 1 for the 2nd dimension proves that it works</span>`}}),nt=new R({}),rt=new B({props:{code:"aW1wb3J0JTIwdGltZSUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBaW1wb3J0JTIwZnVuY3Rvb2xzJTBBJTBBJTIzJTIwdG9yY2glMjBkaXNhYmxlJTIwZ3JhZCUwQXRvcmNoLnNldF9ncmFkX2VuYWJsZWQoRmFsc2UpJTBBJTBBJTIzJTIwc2V0JTIwdmFyaWFibGVzJTBBbl9leHBlcmltZW50cyUyMCUzRCUyMDIlMEF1bmV0X3J1bnNfcGVyX2V4cGVyaW1lbnQlMjAlM0QlMjA1MCUwQSUwQSUwQSUyMyUyMGxvYWQlMjBpbnB1dHMlMEFkZWYlMjBnZW5lcmF0ZV9pbnB1dHMoKSUzQSUwQSUyMCUyMCUyMCUyMHNhbXBsZSUyMCUzRCUyMHRvcmNoLnJhbmRuKDIlMkMlMjA0JTJDJTIwNjQlMkMlMjA2NCkuaGFsZigpLmN1ZGEoKSUwQSUyMCUyMCUyMCUyMHRpbWVzdGVwJTIwJTNEJTIwdG9yY2gucmFuZCgxKS5oYWxmKCkuY3VkYSgpJTIwKiUyMDk5OSUwQSUyMCUyMCUyMCUyMGVuY29kZXJfaGlkZGVuX3N0YXRlcyUyMCUzRCUyMHRvcmNoLnJhbmRuKDIlMkMlMjA3NyUyQyUyMDc2OCkuaGFsZigpLmN1ZGEoKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHNhbXBsZSUyQyUyMHRpbWVzdGVwJTJDJTIwZW5jb2Rlcl9oaWRkZW5fc3RhdGVzJTBBJTBBJTBBcGlwZSUyMCUzRCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJydW53YXltbCUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMHVzZV9zYWZldGVuc29ycyUzRFRydWUlMkMlMEEpLnRvKCUyMmN1ZGElMjIpJTBBdW5ldCUyMCUzRCUyMHBpcGUudW5ldCUwQXVuZXQuZXZhbCgpJTBBdW5ldC50byhtZW1vcnlfZm9ybWF0JTNEdG9yY2guY2hhbm5lbHNfbGFzdCklMjAlMjAlMjMlMjB1c2UlMjBjaGFubmVsc19sYXN0JTIwbWVtb3J5JTIwZm9ybWF0JTBBdW5ldC5mb3J3YXJkJTIwJTNEJTIwZnVuY3Rvb2xzLnBhcnRpYWwodW5ldC5mb3J3YXJkJTJDJTIwcmV0dXJuX2RpY3QlM0RGYWxzZSklMjAlMjAlMjMlMjBzZXQlMjByZXR1cm5fZGljdCUzREZhbHNlJTIwYXMlMjBkZWZhdWx0JTBBJTBBJTIzJTIwd2FybXVwJTBBZm9yJTIwXyUyMGluJTIwcmFuZ2UoMyklM0ElMEElMjAlMjAlMjAlMjB3aXRoJTIwdG9yY2guaW5mZXJlbmNlX21vZGUoKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMGdlbmVyYXRlX2lucHV0cygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3JpZ19vdXRwdXQlMjAlM0QlMjB1bmV0KCppbnB1dHMpJTBBJTBBJTIzJTIwdHJhY2UlMEFwcmludCglMjJ0cmFjaW5nLi4lMjIpJTBBdW5ldF90cmFjZWQlMjAlM0QlMjB0b3JjaC5qaXQudHJhY2UodW5ldCUyQyUyMGlucHV0cyklMEF1bmV0X3RyYWNlZC5ldmFsKCklMEFwcmludCglMjJkb25lJTIwdHJhY2luZyUyMiklMEElMEElMEElMjMlMjB3YXJtdXAlMjBhbmQlMjBvcHRpbWl6ZSUyMGdyYXBoJTBBZm9yJTIwXyUyMGluJTIwcmFuZ2UoNSklM0ElMEElMjAlMjAlMjAlMjB3aXRoJTIwdG9yY2guaW5mZXJlbmNlX21vZGUoKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMGdlbmVyYXRlX2lucHV0cygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3JpZ19vdXRwdXQlMjAlM0QlMjB1bmV0X3RyYWNlZCgqaW5wdXRzKSUwQSUwQSUwQSUyMyUyMGJlbmNobWFya2luZyUwQXdpdGglMjB0b3JjaC5pbmZlcmVuY2VfbW9kZSgpJTNBJTBBJTIwJTIwJTIwJTIwZm9yJTIwXyUyMGluJTIwcmFuZ2Uobl9leHBlcmltZW50cyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3JjaC5jdWRhLnN5bmNocm9uaXplKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdGFydF90aW1lJTIwJTNEJTIwdGltZS50aW1lKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBfJTIwaW4lMjByYW5nZSh1bmV0X3J1bnNfcGVyX2V4cGVyaW1lbnQpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3JpZ19vdXRwdXQlMjAlM0QlMjB1bmV0X3RyYWNlZCgqaW5wdXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvcmNoLmN1ZGEuc3luY2hyb25pemUoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGYlMjJ1bmV0JTIwdHJhY2VkJTIwaW5mZXJlbmNlJTIwdG9vayUyMCU3QnRpbWUudGltZSgpJTIwLSUyMHN0YXJ0X3RpbWUlM0EuMmYlN0QlMjBzZWNvbmRzJTIyKSUwQSUyMCUyMCUyMCUyMGZvciUyMF8lMjBpbiUyMHJhbmdlKG5fZXhwZXJpbWVudHMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdG9yY2guY3VkYS5zeW5jaHJvbml6ZSgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RhcnRfdGltZSUyMCUzRCUyMHRpbWUudGltZSgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwXyUyMGluJTIwcmFuZ2UodW5ldF9ydW5zX3Blcl9leHBlcmltZW50KSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9yaWdfb3V0cHV0JTIwJTNEJTIwdW5ldCgqaW5wdXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvcmNoLmN1ZGEuc3luY2hyb25pemUoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGYlMjJ1bmV0JTIwaW5mZXJlbmNlJTIwdG9vayUyMCU3QnRpbWUudGltZSgpJTIwLSUyMHN0YXJ0X3RpbWUlM0EuMmYlN0QlMjBzZWNvbmRzJTIyKSUwQSUwQSUyMyUyMHNhdmUlMjB0aGUlMjBtb2RlbCUwQXVuZXRfdHJhY2VkLnNhdmUoJTIydW5ldF90cmFjZWQucHQlMjIp",highlighted:`<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> functools

<span class="hljs-comment"># torch disable grad</span>
torch.set_grad_enabled(<span class="hljs-literal">False</span>)

<span class="hljs-comment"># set variables</span>
n_experiments = <span class="hljs-number">2</span>
unet_runs_per_experiment = <span class="hljs-number">50</span>


<span class="hljs-comment"># load inputs</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_inputs</span>():
    sample = torch.randn(<span class="hljs-number">2</span>, <span class="hljs-number">4</span>, <span class="hljs-number">64</span>, <span class="hljs-number">64</span>).half().cuda()
    timestep = torch.rand(<span class="hljs-number">1</span>).half().cuda() * <span class="hljs-number">999</span>
    encoder_hidden_states = torch.randn(<span class="hljs-number">2</span>, <span class="hljs-number">77</span>, <span class="hljs-number">768</span>).half().cuda()
    <span class="hljs-keyword">return</span> sample, timestep, encoder_hidden_states


pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
unet = pipe.unet
unet.<span class="hljs-built_in">eval</span>()
unet.to(memory_format=torch.channels_last)  <span class="hljs-comment"># use channels_last memory format</span>
unet.forward = functools.partial(unet.forward, return_dict=<span class="hljs-literal">False</span>)  <span class="hljs-comment"># set return_dict=False as default</span>

<span class="hljs-comment"># warmup</span>
<span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>):
    <span class="hljs-keyword">with</span> torch.inference_mode():
        inputs = generate_inputs()
        orig_output = unet(*inputs)

<span class="hljs-comment"># trace</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;tracing..&quot;</span>)
unet_traced = torch.jit.trace(unet, inputs)
unet_traced.<span class="hljs-built_in">eval</span>()
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;done tracing&quot;</span>)


<span class="hljs-comment"># warmup and optimize graph</span>
<span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">5</span>):
    <span class="hljs-keyword">with</span> torch.inference_mode():
        inputs = generate_inputs()
        orig_output = unet_traced(*inputs)


<span class="hljs-comment"># benchmarking</span>
<span class="hljs-keyword">with</span> torch.inference_mode():
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_experiments):
        torch.cuda.synchronize()
        start_time = time.time()
        <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(unet_runs_per_experiment):
            orig_output = unet_traced(*inputs)
        torch.cuda.synchronize()
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;unet traced inference took <span class="hljs-subst">{time.time() - start_time:<span class="hljs-number">.2</span>f}</span> seconds&quot;</span>)
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_experiments):
        torch.cuda.synchronize()
        start_time = time.time()
        <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(unet_runs_per_experiment):
            orig_output = unet(*inputs)
        torch.cuda.synchronize()
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;unet inference took <span class="hljs-subst">{time.time() - start_time:<span class="hljs-number">.2</span>f}</span> seconds&quot;</span>)

<span class="hljs-comment"># save the model</span>
unet_traced.save(<span class="hljs-string">&quot;unet_traced.pt&quot;</span>)`}}),it=new B({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGF0YWNsYXNzZXMlMjBpbXBvcnQlMjBkYXRhY2xhc3MlMEElMEElMEElNDBkYXRhY2xhc3MlMEFjbGFzcyUyMFVOZXQyRENvbmRpdGlvbk91dHB1dCUzQSUwQSUyMCUyMCUyMCUyMHNhbXBsZSUzQSUyMHRvcmNoLkZsb2F0VGVuc29yJTBBJTBBJTBBcGlwZSUyMCUzRCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJydW53YXltbCUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMHVzZV9zYWZldGVuc29ycyUzRFRydWUlMkMlMEEpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTIzJTIwdXNlJTIwaml0dGVkJTIwdW5ldCUwQXVuZXRfdHJhY2VkJTIwJTNEJTIwdG9yY2guaml0LmxvYWQoJTIydW5ldF90cmFjZWQucHQlMjIpJTBBJTBBJTBBJTIzJTIwZGVsJTIwcGlwZS51bmV0JTBBY2xhc3MlMjBUcmFjZWRVTmV0KHRvcmNoLm5uLk1vZHVsZSklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN1cGVyKCkuX19pbml0X18oKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuaW5fY2hhbm5lbHMlMjAlM0QlMjBwaXBlLnVuZXQuaW5fY2hhbm5lbHMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmRldmljZSUyMCUzRCUyMHBpcGUudW5ldC5kZXZpY2UlMEElMEElMjAlMjAlMjAlMjBkZWYlMjBmb3J3YXJkKHNlbGYlMkMlMjBsYXRlbnRfbW9kZWxfaW5wdXQlMkMlMjB0JTJDJTIwZW5jb2Rlcl9oaWRkZW5fc3RhdGVzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNhbXBsZSUyMCUzRCUyMHVuZXRfdHJhY2VkKGxhdGVudF9tb2RlbF9pbnB1dCUyQyUyMHQlMkMlMjBlbmNvZGVyX2hpZGRlbl9zdGF0ZXMpJTVCMCU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMFVOZXQyRENvbmRpdGlvbk91dHB1dChzYW1wbGUlM0RzYW1wbGUpJTBBJTBBJTBBcGlwZS51bmV0JTIwJTNEJTIwVHJhY2VkVU5ldCgpJTBBJTBBd2l0aCUyMHRvcmNoLmluZmVyZW5jZV9tb2RlKCklM0ElMEElMjAlMjAlMjAlMjBpbWFnZSUyMCUzRCUyMHBpcGUoJTVCcHJvbXB0JTVEJTIwKiUyMDElMkMlMjBudW1faW5mZXJlbmNlX3N0ZXBzJTNENTApLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> dataclasses <span class="hljs-keyword">import</span> dataclass


<span class="hljs-meta">@dataclass</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">UNet2DConditionOutput</span>:
    sample: torch.FloatTensor


pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># use jitted unet</span>
unet_traced = torch.jit.load(<span class="hljs-string">&quot;unet_traced.pt&quot;</span>)


<span class="hljs-comment"># del pipe.unet</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">TracedUNet</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.in_channels = pipe.unet.in_channels
        self.device = pipe.unet.device

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, latent_model_input, t, encoder_hidden_states</span>):
        sample = unet_traced(latent_model_input, t, encoder_hidden_states)[<span class="hljs-number">0</span>]
        <span class="hljs-keyword">return</span> UNet2DConditionOutput(sample=sample)


pipe.unet = TracedUNet()

<span class="hljs-keyword">with</span> torch.inference_mode():
    image = pipe([prompt] * <span class="hljs-number">1</span>, num_inference_steps=<span class="hljs-number">50</span>).images[<span class="hljs-number">0</span>]`}}),pt=new R({}),$e=new uo({props:{warning:!0,$$slots:{default:[uf]},$$scope:{ctx:W}}}),ct=new B({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMERpZmZ1c2lvblBpcGVsaW5lJTBBaW1wb3J0JTIwdG9yY2glMEElMEFwaXBlJTIwJTNEJTIwRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSkudG8oJTIyY3VkYSUyMiklMEElMEFwaXBlLmVuYWJsZV94Zm9ybWVyc19tZW1vcnlfZWZmaWNpZW50X2F0dGVudGlvbigpJTBBJTBBd2l0aCUyMHRvcmNoLmluZmVyZW5jZV9tb2RlKCklM0ElMEElMjAlMjAlMjAlMjBzYW1wbGUlMjAlM0QlMjBwaXBlKCUyMmElMjBzbWFsbCUyMGNhdCUyMiklMEElMEElMjMlMjBvcHRpb25hbCUzQSUyMFlvdSUyMGNhbiUyMGRpc2FibGUlMjBpdCUyMHZpYSUwQSUyMyUyMHBpcGUuZGlzYWJsZV94Zm9ybWVyc19tZW1vcnlfZWZmaWNpZW50X2F0dGVudGlvbigp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">import</span> torch

pipe = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

pipe.enable_xformers_memory_efficient_attention()

<span class="hljs-keyword">with</span> torch.inference_mode():
    sample = pipe(<span class="hljs-string">&quot;a small cat&quot;</span>)

<span class="hljs-comment"># optional: You can disable it via</span>
<span class="hljs-comment"># pipe.disable_xformers_memory_efficient_attention()</span>`}}),{c(){d=s("meta"),u=m(),h=s("h1"),U=s("a"),v=s("span"),y(_.$$.fragment),S=m(),Z=s("span"),ut=n("Memory and speed"),Ne=m(),C=s("p"),yo=n("We present some techniques and ideas to optimize \u{1F917} Diffusers "),rl=s("em"),wo=n("inference"),Mo=n(" for memory or speed. As a general rule, we recommend the use of "),Re=s("a"),Jo=n("xFormers"),To=n(" for memory efficient attention, please see the recommended "),yt=s("a"),bo=n("installation instructions"),Uo=n("."),$s=m(),wt=s("p"),vo=n("We\u2019ll discuss how the following settings impact performance and memory."),As=m(),ce=s("table"),il=s("thead"),X=s("tr"),Ns=s("th"),_o=m(),pl=s("th"),jo=n("Latency"),Eo=m(),ml=s("th"),Io=n("Speedup"),go=m(),I=s("tbody"),Y=s("tr"),fl=s("td"),Zo=n("original"),Co=m(),cl=s("td"),Bo=n("9.50s"),Go=m(),dl=s("td"),Wo=n("x1"),ko=m(),Q=s("tr"),hl=s("td"),Vo=n("fp16"),$o=m(),ul=s("td"),Ao=n("3.61s"),No=m(),yl=s("td"),Ro=n("x2.63"),So=m(),z=s("tr"),wl=s("td"),Xo=n("channels last"),Yo=m(),Ml=s("td"),Qo=n("3.30s"),zo=m(),Jl=s("td"),xo=n("x2.88"),Fo=m(),x=s("tr"),Tl=s("td"),Ho=n("traced UNet"),Do=m(),bl=s("td"),Po=n("3.21s"),qo=m(),Ul=s("td"),Lo=n("x2.96"),Ko=m(),F=s("tr"),vl=s("td"),Oo=n("memory efficient attention"),en=m(),_l=s("td"),tn=n("2.63s"),ln=m(),jl=s("td"),sn=n("x3.61"),Rs=m(),Mt=s("em"),an=n(`obtained on NVIDIA TITAN RTX by generating a single image of size 512x512 from
  the prompt "a photo of an astronaut riding a horse on mars" with 50 DDIM
  steps.`),Ss=m(),H=s("h3"),de=s("a"),El=s("span"),y(Se.$$.fragment),on=m(),Il=s("span"),nn=n("Use tf32 instead of fp32 (on Ampere and later CUDA devices)"),Xs=m(),he=s("p"),rn=n("On Ampere and later CUDA devices matrix multiplications and convolutions can use the TensorFloat32 (TF32) mode for faster but slightly less accurate computations. By default PyTorch enables TF32 mode for convolutions but not matrix multiplications, and unless a network requires full float32 precision we recommend enabling this setting for matrix multiplications, too. It can significantly speed up computations with typically negligible loss of numerical accuracy. You can read more about it "),Xe=s("a"),pn=n("here"),mn=n(". All you need to do is to add this before your inference:"),Ys=m(),y(Ye.$$.fragment),Qs=m(),D=s("h2"),ue=s("a"),gl=s("span"),y(Qe.$$.fragment),fn=m(),Zl=s("span"),cn=n("Half precision weights"),zs=m(),k=s("p"),dn=n("To save more GPU memory and get more speed, you can load and run the model weights directly in half precision. This involves loading the float16 version of the weights, which was saved to a branch named "),Cl=s("code"),hn=n("fp16"),un=n(", and telling PyTorch to use the "),Bl=s("code"),yn=n("float16"),wn=n(" type when loading them:"),xs=m(),y(ze.$$.fragment),Fs=m(),y(ye.$$.fragment),Hs=m(),P=s("h2"),we=s("a"),Gl=s("span"),y(xe.$$.fragment),Mn=m(),Wl=s("span"),Jn=n("Sliced VAE decode for larger batches"),Ds=m(),Jt=s("p"),Tn=n("To decode large batches of images with limited VRAM, or to enable batches with 32 images or more, you can use sliced VAE decode that decodes the batch latents one image at a time."),Ps=m(),Me=s("p"),bn=n("You likely want to couple this with "),Tt=s("a"),Un=n("enable_xformers_memory_efficient_attention()"),vn=n(" to further minimize memory use."),qs=m(),Je=s("p"),_n=n("To perform the VAE decode one image at a time, invoke "),bt=s("a"),jn=n("enable_vae_slicing()"),En=n(" in your pipeline before inference. For example:"),Ls=m(),y(Fe.$$.fragment),Ks=m(),Ut=s("p"),In=n("You may see a small performance boost in VAE decode on multi-image batches. There should be no performance impact on single-image batches."),Os=m(),q=s("h2"),Te=s("a"),kl=s("span"),y(He.$$.fragment),gn=m(),Vl=s("span"),Zn=n("Tiled VAE decode and encode for large images"),ea=m(),vt=s("p"),Cn=n("Tiled VAE processing makes it possible to work with large images on limited VRAM. For example, generating 4k images in 8GB of VRAM. Tiled VAE decoder splits the image into overlapping tiles, decodes the tiles, and blends the outputs to make the final image."),ta=m(),be=s("p"),Bn=n("You want to couple this with "),_t=s("a"),Gn=n("enable_xformers_memory_efficient_attention()"),Wn=n(" to further minimize memory use."),la=m(),Ue=s("p"),kn=n("To use tiled VAE processing, invoke "),jt=s("a"),Vn=n("enable_vae_tiling()"),$n=n(" in your pipeline before inference. For example:"),sa=m(),y(De.$$.fragment),aa=m(),Et=s("p"),An=n("The output image will have some tile-to-tile tone variation from the tiles having separate decoders, but you shouldn\u2019t see sharp seams between the tiles. The tiling is turned off for images that are 512x512 or smaller."),oa=m(),It=s("a"),na=m(),L=s("h2"),ve=s("a"),$l=s("span"),y(Pe.$$.fragment),Nn=m(),Al=s("span"),Rn=n("Offloading to CPU with accelerate for memory savings"),ra=m(),gt=s("p"),Sn=n("For additional memory savings, you can offload the weights to CPU and only load them to GPU when performing the forward pass."),ia=m(),_e=s("p"),Xn=n("To perform CPU offloading, all you have to do is invoke "),Zt=s("a"),Yn=n("enable_sequential_cpu_offload()"),Qn=n(":"),pa=m(),y(qe.$$.fragment),ma=m(),Ct=s("p"),zn=n("And you can get the memory consumption to < 3GB."),fa=m(),je=s("p"),xn=n("Note that this method works at the submodule level, not on whole models. This is the best way to minimize memory consumption, but inference is much slower due to the iterative nature of the process. The UNet component of the pipeline runs several times (as many as "),Nl=s("code"),Fn=n("num_inference_steps"),Hn=n("); each time, the different submodules of the UNet are sequentially onloaded and then offloaded as they are needed, so the number of memory transfers is large."),ca=m(),y(Ee.$$.fragment),da=m(),Bt=s("p"),Dn=n("It is also possible to chain offloading with attention slicing for minimal memory consumption (< 2GB)."),ha=m(),y(Le.$$.fragment),ua=m(),g=s("p"),Rl=s("strong"),Pn=n("Note"),qn=n(": When using "),Sl=s("code"),Ln=n("enable_sequential_cpu_offload()"),Kn=n(", it is important to "),Xl=s("strong"),On=n("not"),er=n(" move the pipeline to CUDA beforehand or else the gain in memory consumption will only be minimal. See "),Ke=s("a"),tr=n("this issue"),lr=n(" for more information."),ya=m(),K=s("p"),Yl=s("strong"),sr=n("Note"),ar=n(": "),Ql=s("code"),or=n("enable_sequential_cpu_offload()"),nr=n(" is a stateful operation that installs hooks on the models."),wa=m(),Gt=s("a"),Ma=m(),O=s("h2"),Ie=s("a"),zl=s("span"),y(Oe.$$.fragment),rr=m(),xl=s("span"),ir=n("Model offloading for fast inference and memory savings"),Ja=m(),et=s("p"),Wt=s("a"),pr=n("Sequential CPU offloading"),mr=n(", as discussed in the previous section, preserves a lot of memory but makes inference slower, because submodules are moved to GPU as needed, and immediately returned to CPU when a new module runs."),Ta=m(),V=s("p"),fr=n("Full-model offloading is an alternative that moves whole models to the GPU, instead of handling each model\u2019s constituent "),Fl=s("em"),cr=n("modules"),dr=n(". This results in a negligible impact on inference time (compared with moving the pipeline to "),Hl=s("code"),hr=n("cuda"),ur=n("), while still providing some memory savings."),ba=m(),kt=s("p"),yr=n(`In this scenario, only one of the main components of the pipeline (typically: text encoder, unet and vae)
will be in the GPU while the others wait in the CPU. Components like the UNet that run for multiple iterations will stay on GPU until they are no longer needed.`),Ua=m(),ge=s("p"),wr=n("This feature can be enabled by invoking "),Dl=s("code"),Mr=n("enable_model_cpu_offload()"),Jr=n(" on the pipeline, as shown below."),va=m(),y(tt.$$.fragment),_a=m(),Vt=s("p"),Tr=n("This is also compatible with attention slicing for additional memory savings."),ja=m(),y(lt.$$.fragment),Ea=m(),y(Ze.$$.fragment),Ia=m(),G=s("p"),Pl=s("strong"),br=n("Note"),Ur=n(": "),ql=s("code"),vr=n("enable_model_cpu_offload()"),_r=n(` is a stateful operation that installs hooks on the models and state on the pipeline. In order to properly offload
models after they are called, it is required that the entire pipeline is run and models are called in the order the pipeline expects them to be. Exercise caution
if models are re-used outside the context of the pipeline after hooks have been installed. See `),st=s("a"),jr=n("accelerate"),Er=n(`
for further docs on removing hooks.`),ga=m(),ee=s("h2"),Ce=s("a"),Ll=s("span"),y(at.$$.fragment),Ir=m(),Kl=s("span"),gr=n("Using Channels Last memory format"),Za=m(),$t=s("p"),Zr=n("Channels last memory format is an alternative way of ordering NCHW tensors in memory preserving dimensions ordering. Channels last tensors ordered in such a way that channels become the densest dimension (aka storing images pixel-per-pixel). Since not all operators currently support channels last format it may result in a worst performance, so it\u2019s better to try it and see if it works for your model."),Ca=m(),At=s("p"),Cr=n("For example, in order to set the UNet model in our pipeline to use channels last format, we can use the following:"),Ba=m(),y(ot.$$.fragment),Ga=m(),te=s("h2"),Be=s("a"),Ol=s("span"),y(nt.$$.fragment),Br=m(),es=s("span"),Gr=n("Tracing"),Wa=m(),Ge=s("p"),Wr=n("Tracing runs an example input tensor through your model, and captures the operations that are invoked as that input makes its way through the model\u2019s layers so that an executable or "),ts=s("code"),kr=n("ScriptFunction"),Vr=n(" is returned that will be optimized using just-in-time compilation."),ka=m(),Nt=s("p"),$r=n("To trace our UNet model, we can use the following:"),Va=m(),y(rt.$$.fragment),$a=m(),We=s("p"),Ar=n("Then we can replace the "),ls=s("code"),Nr=n("unet"),Rr=n(" attribute of the pipeline with the traced model like the following"),Aa=m(),y(it.$$.fragment),Na=m(),le=s("h2"),ke=s("a"),ss=s("span"),y(pt.$$.fragment),Sr=m(),as=s("span"),Xr=n("Memory Efficient Attention"),Ra=m(),$=s("p"),Yr=n("Recent work on optimizing the bandwitdh in the attention block has generated huge speed ups and gains in GPU memory usage. The most recent being Flash Attention from @tridao: "),mt=s("a"),Qr=n("code"),zr=n(", "),ft=s("a"),xr=n("paper"),Fr=n("."),Sa=m(),Rt=s("p"),Hr=n("Here are the speedups we obtain on a few Nvidia GPUs when running the inference at 512x512 with a batch size of 1 (one prompt):"),Xa=m(),Ve=s("table"),os=s("thead"),se=s("tr"),ns=s("th"),Dr=n("GPU"),Pr=m(),rs=s("th"),qr=n("Base Attention FP16"),Lr=m(),is=s("th"),Kr=n("Memory Efficient Attention FP16"),Or=m(),j=s("tbody"),ae=s("tr"),ps=s("td"),ei=n("NVIDIA Tesla T4"),ti=m(),ms=s("td"),li=n("3.5it/s"),si=m(),fs=s("td"),ai=n("5.5it/s"),oi=m(),oe=s("tr"),cs=s("td"),ni=n("NVIDIA 3060 RTX"),ri=m(),ds=s("td"),ii=n("4.6it/s"),pi=m(),hs=s("td"),mi=n("7.8it/s"),fi=m(),ne=s("tr"),us=s("td"),ci=n("NVIDIA A10G"),di=m(),ys=s("td"),hi=n("8.88it/s"),ui=m(),ws=s("td"),yi=n("15.6it/s"),wi=m(),re=s("tr"),Ms=s("td"),Mi=n("NVIDIA RTX A6000"),Ji=m(),Js=s("td"),Ti=n("11.7it/s"),bi=m(),Ts=s("td"),Ui=n("21.09it/s"),vi=m(),ie=s("tr"),bs=s("td"),_i=n("NVIDIA TITAN RTX"),ji=m(),Us=s("td"),Ei=n("12.51it/s"),Ii=m(),vs=s("td"),gi=n("18.22it/s"),Zi=m(),pe=s("tr"),_s=s("td"),Ci=n("A100-SXM4-40GB"),Bi=m(),js=s("td"),Gi=n("18.6it/s"),Wi=m(),Es=s("td"),ki=n("29.it/s"),Vi=m(),me=s("tr"),Is=s("td"),$i=n("A100-SXM-80GB"),Ai=m(),gs=s("td"),Ni=n("18.7it/s"),Ri=m(),Zs=s("td"),Si=n("29.5it/s"),Ya=m(),St=s("p"),Xi=n("To leverage it just make sure you have:"),Qa=m(),y($e.$$.fragment),za=m(),A=s("ul"),Cs=s("li"),Yi=n("PyTorch > 1.12"),Qi=m(),Bs=s("li"),zi=n("Cuda available"),xi=m(),Xt=s("li"),Yt=s("a"),Fi=n("Installed the xformers library"),Hi=n("."),xa=m(),y(ct.$$.fragment),this.h()},l(e){const i=mf('[data-svelte="svelte-1phssyn"]',document.head);d=a(i,"META",{name:!0,content:!0}),i.forEach(l),u=f(e),h=a(e,"H1",{class:!0});var dt=o(h);U=a(dt,"A",{id:!0,class:!0,href:!0});var Gs=o(U);v=a(Gs,"SPAN",{});var Ws=o(v);w(_.$$.fragment,Ws),Ws.forEach(l),Gs.forEach(l),S=f(dt),Z=a(dt,"SPAN",{});var ks=o(Z);ut=r(ks,"Memory and speed"),ks.forEach(l),dt.forEach(l),Ne=f(e),C=a(e,"P",{});var Ae=o(C);yo=r(Ae,"We present some techniques and ideas to optimize \u{1F917} Diffusers "),rl=a(Ae,"EM",{});var qi=o(rl);wo=r(qi,"inference"),qi.forEach(l),Mo=r(Ae," for memory or speed. As a general rule, we recommend the use of "),Re=a(Ae,"A",{href:!0,rel:!0});var Li=o(Re);Jo=r(Li,"xFormers"),Li.forEach(l),To=r(Ae," for memory efficient attention, please see the recommended "),yt=a(Ae,"A",{href:!0});var Ki=o(yt);bo=r(Ki,"installation instructions"),Ki.forEach(l),Uo=r(Ae,"."),Ae.forEach(l),$s=f(e),wt=a(e,"P",{});var Oi=o(wt);vo=r(Oi,"We\u2019ll discuss how the following settings impact performance and memory."),Oi.forEach(l),As=f(e),ce=a(e,"TABLE",{});var Ha=o(ce);il=a(Ha,"THEAD",{});var ep=o(il);X=a(ep,"TR",{});var Qt=o(X);Ns=a(Qt,"TH",{}),o(Ns).forEach(l),_o=f(Qt),pl=a(Qt,"TH",{});var tp=o(pl);jo=r(tp,"Latency"),tp.forEach(l),Eo=f(Qt),ml=a(Qt,"TH",{});var lp=o(ml);Io=r(lp,"Speedup"),lp.forEach(l),Qt.forEach(l),ep.forEach(l),go=f(Ha),I=a(Ha,"TBODY",{});var N=o(I);Y=a(N,"TR",{});var zt=o(Y);fl=a(zt,"TD",{});var sp=o(fl);Zo=r(sp,"original"),sp.forEach(l),Co=f(zt),cl=a(zt,"TD",{});var ap=o(cl);Bo=r(ap,"9.50s"),ap.forEach(l),Go=f(zt),dl=a(zt,"TD",{});var op=o(dl);Wo=r(op,"x1"),op.forEach(l),zt.forEach(l),ko=f(N),Q=a(N,"TR",{});var xt=o(Q);hl=a(xt,"TD",{});var np=o(hl);Vo=r(np,"fp16"),np.forEach(l),$o=f(xt),ul=a(xt,"TD",{});var rp=o(ul);Ao=r(rp,"3.61s"),rp.forEach(l),No=f(xt),yl=a(xt,"TD",{});var ip=o(yl);Ro=r(ip,"x2.63"),ip.forEach(l),xt.forEach(l),So=f(N),z=a(N,"TR",{});var Ft=o(z);wl=a(Ft,"TD",{});var pp=o(wl);Xo=r(pp,"channels last"),pp.forEach(l),Yo=f(Ft),Ml=a(Ft,"TD",{});var mp=o(Ml);Qo=r(mp,"3.30s"),mp.forEach(l),zo=f(Ft),Jl=a(Ft,"TD",{});var fp=o(Jl);xo=r(fp,"x2.88"),fp.forEach(l),Ft.forEach(l),Fo=f(N),x=a(N,"TR",{});var Ht=o(x);Tl=a(Ht,"TD",{});var cp=o(Tl);Ho=r(cp,"traced UNet"),cp.forEach(l),Do=f(Ht),bl=a(Ht,"TD",{});var dp=o(bl);Po=r(dp,"3.21s"),dp.forEach(l),qo=f(Ht),Ul=a(Ht,"TD",{});var hp=o(Ul);Lo=r(hp,"x2.96"),hp.forEach(l),Ht.forEach(l),Ko=f(N),F=a(N,"TR",{});var Dt=o(F);vl=a(Dt,"TD",{});var up=o(vl);Oo=r(up,"memory efficient attention"),up.forEach(l),en=f(Dt),_l=a(Dt,"TD",{});var yp=o(_l);tn=r(yp,"2.63s"),yp.forEach(l),ln=f(Dt),jl=a(Dt,"TD",{});var wp=o(jl);sn=r(wp,"x3.61"),wp.forEach(l),Dt.forEach(l),N.forEach(l),Ha.forEach(l),Rs=f(e),Mt=a(e,"EM",{});var Mp=o(Mt);an=r(Mp,`obtained on NVIDIA TITAN RTX by generating a single image of size 512x512 from
  the prompt "a photo of an astronaut riding a horse on mars" with 50 DDIM
  steps.`),Mp.forEach(l),Ss=f(e),H=a(e,"H3",{class:!0});var Da=o(H);de=a(Da,"A",{id:!0,class:!0,href:!0});var Jp=o(de);El=a(Jp,"SPAN",{});var Tp=o(El);w(Se.$$.fragment,Tp),Tp.forEach(l),Jp.forEach(l),on=f(Da),Il=a(Da,"SPAN",{});var bp=o(Il);nn=r(bp,"Use tf32 instead of fp32 (on Ampere and later CUDA devices)"),bp.forEach(l),Da.forEach(l),Xs=f(e),he=a(e,"P",{});var Pa=o(he);rn=r(Pa,"On Ampere and later CUDA devices matrix multiplications and convolutions can use the TensorFloat32 (TF32) mode for faster but slightly less accurate computations. By default PyTorch enables TF32 mode for convolutions but not matrix multiplications, and unless a network requires full float32 precision we recommend enabling this setting for matrix multiplications, too. It can significantly speed up computations with typically negligible loss of numerical accuracy. You can read more about it "),Xe=a(Pa,"A",{href:!0,rel:!0});var Up=o(Xe);pn=r(Up,"here"),Up.forEach(l),mn=r(Pa,". All you need to do is to add this before your inference:"),Pa.forEach(l),Ys=f(e),w(Ye.$$.fragment,e),Qs=f(e),D=a(e,"H2",{class:!0});var qa=o(D);ue=a(qa,"A",{id:!0,class:!0,href:!0});var vp=o(ue);gl=a(vp,"SPAN",{});var _p=o(gl);w(Qe.$$.fragment,_p),_p.forEach(l),vp.forEach(l),fn=f(qa),Zl=a(qa,"SPAN",{});var jp=o(Zl);cn=r(jp,"Half precision weights"),jp.forEach(l),qa.forEach(l),zs=f(e),k=a(e,"P",{});var Pt=o(k);dn=r(Pt,"To save more GPU memory and get more speed, you can load and run the model weights directly in half precision. This involves loading the float16 version of the weights, which was saved to a branch named "),Cl=a(Pt,"CODE",{});var Ep=o(Cl);hn=r(Ep,"fp16"),Ep.forEach(l),un=r(Pt,", and telling PyTorch to use the "),Bl=a(Pt,"CODE",{});var Ip=o(Bl);yn=r(Ip,"float16"),Ip.forEach(l),wn=r(Pt," type when loading them:"),Pt.forEach(l),xs=f(e),w(ze.$$.fragment,e),Fs=f(e),w(ye.$$.fragment,e),Hs=f(e),P=a(e,"H2",{class:!0});var La=o(P);we=a(La,"A",{id:!0,class:!0,href:!0});var gp=o(we);Gl=a(gp,"SPAN",{});var Zp=o(Gl);w(xe.$$.fragment,Zp),Zp.forEach(l),gp.forEach(l),Mn=f(La),Wl=a(La,"SPAN",{});var Cp=o(Wl);Jn=r(Cp,"Sliced VAE decode for larger batches"),Cp.forEach(l),La.forEach(l),Ds=f(e),Jt=a(e,"P",{});var Bp=o(Jt);Tn=r(Bp,"To decode large batches of images with limited VRAM, or to enable batches with 32 images or more, you can use sliced VAE decode that decodes the batch latents one image at a time."),Bp.forEach(l),Ps=f(e),Me=a(e,"P",{});var Ka=o(Me);bn=r(Ka,"You likely want to couple this with "),Tt=a(Ka,"A",{href:!0});var Gp=o(Tt);Un=r(Gp,"enable_xformers_memory_efficient_attention()"),Gp.forEach(l),vn=r(Ka," to further minimize memory use."),Ka.forEach(l),qs=f(e),Je=a(e,"P",{});var Oa=o(Je);_n=r(Oa,"To perform the VAE decode one image at a time, invoke "),bt=a(Oa,"A",{href:!0});var Wp=o(bt);jn=r(Wp,"enable_vae_slicing()"),Wp.forEach(l),En=r(Oa," in your pipeline before inference. For example:"),Oa.forEach(l),Ls=f(e),w(Fe.$$.fragment,e),Ks=f(e),Ut=a(e,"P",{});var kp=o(Ut);In=r(kp,"You may see a small performance boost in VAE decode on multi-image batches. There should be no performance impact on single-image batches."),kp.forEach(l),Os=f(e),q=a(e,"H2",{class:!0});var eo=o(q);Te=a(eo,"A",{id:!0,class:!0,href:!0});var Vp=o(Te);kl=a(Vp,"SPAN",{});var $p=o(kl);w(He.$$.fragment,$p),$p.forEach(l),Vp.forEach(l),gn=f(eo),Vl=a(eo,"SPAN",{});var Ap=o(Vl);Zn=r(Ap,"Tiled VAE decode and encode for large images"),Ap.forEach(l),eo.forEach(l),ea=f(e),vt=a(e,"P",{});var Np=o(vt);Cn=r(Np,"Tiled VAE processing makes it possible to work with large images on limited VRAM. For example, generating 4k images in 8GB of VRAM. Tiled VAE decoder splits the image into overlapping tiles, decodes the tiles, and blends the outputs to make the final image."),Np.forEach(l),ta=f(e),be=a(e,"P",{});var to=o(be);Bn=r(to,"You want to couple this with "),_t=a(to,"A",{href:!0});var Rp=o(_t);Gn=r(Rp,"enable_xformers_memory_efficient_attention()"),Rp.forEach(l),Wn=r(to," to further minimize memory use."),to.forEach(l),la=f(e),Ue=a(e,"P",{});var lo=o(Ue);kn=r(lo,"To use tiled VAE processing, invoke "),jt=a(lo,"A",{href:!0});var Sp=o(jt);Vn=r(Sp,"enable_vae_tiling()"),Sp.forEach(l),$n=r(lo," in your pipeline before inference. For example:"),lo.forEach(l),sa=f(e),w(De.$$.fragment,e),aa=f(e),Et=a(e,"P",{});var Xp=o(Et);An=r(Xp,"The output image will have some tile-to-tile tone variation from the tiles having separate decoders, but you shouldn\u2019t see sharp seams between the tiles. The tiling is turned off for images that are 512x512 or smaller."),Xp.forEach(l),oa=f(e),It=a(e,"A",{name:!0}),o(It).forEach(l),na=f(e),L=a(e,"H2",{class:!0});var so=o(L);ve=a(so,"A",{id:!0,class:!0,href:!0});var Yp=o(ve);$l=a(Yp,"SPAN",{});var Qp=o($l);w(Pe.$$.fragment,Qp),Qp.forEach(l),Yp.forEach(l),Nn=f(so),Al=a(so,"SPAN",{});var zp=o(Al);Rn=r(zp,"Offloading to CPU with accelerate for memory savings"),zp.forEach(l),so.forEach(l),ra=f(e),gt=a(e,"P",{});var xp=o(gt);Sn=r(xp,"For additional memory savings, you can offload the weights to CPU and only load them to GPU when performing the forward pass."),xp.forEach(l),ia=f(e),_e=a(e,"P",{});var ao=o(_e);Xn=r(ao,"To perform CPU offloading, all you have to do is invoke "),Zt=a(ao,"A",{href:!0});var Fp=o(Zt);Yn=r(Fp,"enable_sequential_cpu_offload()"),Fp.forEach(l),Qn=r(ao,":"),ao.forEach(l),pa=f(e),w(qe.$$.fragment,e),ma=f(e),Ct=a(e,"P",{});var Hp=o(Ct);zn=r(Hp,"And you can get the memory consumption to < 3GB."),Hp.forEach(l),fa=f(e),je=a(e,"P",{});var oo=o(je);xn=r(oo,"Note that this method works at the submodule level, not on whole models. This is the best way to minimize memory consumption, but inference is much slower due to the iterative nature of the process. The UNet component of the pipeline runs several times (as many as "),Nl=a(oo,"CODE",{});var Dp=o(Nl);Fn=r(Dp,"num_inference_steps"),Dp.forEach(l),Hn=r(oo,"); each time, the different submodules of the UNet are sequentially onloaded and then offloaded as they are needed, so the number of memory transfers is large."),oo.forEach(l),ca=f(e),w(Ee.$$.fragment,e),da=f(e),Bt=a(e,"P",{});var Pp=o(Bt);Dn=r(Pp,"It is also possible to chain offloading with attention slicing for minimal memory consumption (< 2GB)."),Pp.forEach(l),ha=f(e),w(Le.$$.fragment,e),ua=f(e),g=a(e,"P",{});var fe=o(g);Rl=a(fe,"STRONG",{});var qp=o(Rl);Pn=r(qp,"Note"),qp.forEach(l),qn=r(fe,": When using "),Sl=a(fe,"CODE",{});var Lp=o(Sl);Ln=r(Lp,"enable_sequential_cpu_offload()"),Lp.forEach(l),Kn=r(fe,", it is important to "),Xl=a(fe,"STRONG",{});var Kp=o(Xl);On=r(Kp,"not"),Kp.forEach(l),er=r(fe," move the pipeline to CUDA beforehand or else the gain in memory consumption will only be minimal. See "),Ke=a(fe,"A",{href:!0,rel:!0});var Op=o(Ke);tr=r(Op,"this issue"),Op.forEach(l),lr=r(fe," for more information."),fe.forEach(l),ya=f(e),K=a(e,"P",{});var Vs=o(K);Yl=a(Vs,"STRONG",{});var em=o(Yl);sr=r(em,"Note"),em.forEach(l),ar=r(Vs,": "),Ql=a(Vs,"CODE",{});var tm=o(Ql);or=r(tm,"enable_sequential_cpu_offload()"),tm.forEach(l),nr=r(Vs," is a stateful operation that installs hooks on the models."),Vs.forEach(l),wa=f(e),Gt=a(e,"A",{name:!0}),o(Gt).forEach(l),Ma=f(e),O=a(e,"H2",{class:!0});var no=o(O);Ie=a(no,"A",{id:!0,class:!0,href:!0});var lm=o(Ie);zl=a(lm,"SPAN",{});var sm=o(zl);w(Oe.$$.fragment,sm),sm.forEach(l),lm.forEach(l),rr=f(no),xl=a(no,"SPAN",{});var am=o(xl);ir=r(am,"Model offloading for fast inference and memory savings"),am.forEach(l),no.forEach(l),Ja=f(e),et=a(e,"P",{});var Di=o(et);Wt=a(Di,"A",{href:!0});var om=o(Wt);pr=r(om,"Sequential CPU offloading"),om.forEach(l),mr=r(Di,", as discussed in the previous section, preserves a lot of memory but makes inference slower, because submodules are moved to GPU as needed, and immediately returned to CPU when a new module runs."),Di.forEach(l),Ta=f(e),V=a(e,"P",{});var qt=o(V);fr=r(qt,"Full-model offloading is an alternative that moves whole models to the GPU, instead of handling each model\u2019s constituent "),Fl=a(qt,"EM",{});var nm=o(Fl);cr=r(nm,"modules"),nm.forEach(l),dr=r(qt,". This results in a negligible impact on inference time (compared with moving the pipeline to "),Hl=a(qt,"CODE",{});var rm=o(Hl);hr=r(rm,"cuda"),rm.forEach(l),ur=r(qt,"), while still providing some memory savings."),qt.forEach(l),ba=f(e),kt=a(e,"P",{});var im=o(kt);yr=r(im,`In this scenario, only one of the main components of the pipeline (typically: text encoder, unet and vae)
will be in the GPU while the others wait in the CPU. Components like the UNet that run for multiple iterations will stay on GPU until they are no longer needed.`),im.forEach(l),Ua=f(e),ge=a(e,"P",{});var ro=o(ge);wr=r(ro,"This feature can be enabled by invoking "),Dl=a(ro,"CODE",{});var pm=o(Dl);Mr=r(pm,"enable_model_cpu_offload()"),pm.forEach(l),Jr=r(ro," on the pipeline, as shown below."),ro.forEach(l),va=f(e),w(tt.$$.fragment,e),_a=f(e),Vt=a(e,"P",{});var mm=o(Vt);Tr=r(mm,"This is also compatible with attention slicing for additional memory savings."),mm.forEach(l),ja=f(e),w(lt.$$.fragment,e),Ea=f(e),w(Ze.$$.fragment,e),Ia=f(e),G=a(e,"P",{});var ht=o(G);Pl=a(ht,"STRONG",{});var fm=o(Pl);br=r(fm,"Note"),fm.forEach(l),Ur=r(ht,": "),ql=a(ht,"CODE",{});var cm=o(ql);vr=r(cm,"enable_model_cpu_offload()"),cm.forEach(l),_r=r(ht,` is a stateful operation that installs hooks on the models and state on the pipeline. In order to properly offload
models after they are called, it is required that the entire pipeline is run and models are called in the order the pipeline expects them to be. Exercise caution
if models are re-used outside the context of the pipeline after hooks have been installed. See `),st=a(ht,"A",{href:!0,rel:!0});var dm=o(st);jr=r(dm,"accelerate"),dm.forEach(l),Er=r(ht,`
for further docs on removing hooks.`),ht.forEach(l),ga=f(e),ee=a(e,"H2",{class:!0});var io=o(ee);Ce=a(io,"A",{id:!0,class:!0,href:!0});var hm=o(Ce);Ll=a(hm,"SPAN",{});var um=o(Ll);w(at.$$.fragment,um),um.forEach(l),hm.forEach(l),Ir=f(io),Kl=a(io,"SPAN",{});var ym=o(Kl);gr=r(ym,"Using Channels Last memory format"),ym.forEach(l),io.forEach(l),Za=f(e),$t=a(e,"P",{});var wm=o($t);Zr=r(wm,"Channels last memory format is an alternative way of ordering NCHW tensors in memory preserving dimensions ordering. Channels last tensors ordered in such a way that channels become the densest dimension (aka storing images pixel-per-pixel). Since not all operators currently support channels last format it may result in a worst performance, so it\u2019s better to try it and see if it works for your model."),wm.forEach(l),Ca=f(e),At=a(e,"P",{});var Mm=o(At);Cr=r(Mm,"For example, in order to set the UNet model in our pipeline to use channels last format, we can use the following:"),Mm.forEach(l),Ba=f(e),w(ot.$$.fragment,e),Ga=f(e),te=a(e,"H2",{class:!0});var po=o(te);Be=a(po,"A",{id:!0,class:!0,href:!0});var Jm=o(Be);Ol=a(Jm,"SPAN",{});var Tm=o(Ol);w(nt.$$.fragment,Tm),Tm.forEach(l),Jm.forEach(l),Br=f(po),es=a(po,"SPAN",{});var bm=o(es);Gr=r(bm,"Tracing"),bm.forEach(l),po.forEach(l),Wa=f(e),Ge=a(e,"P",{});var mo=o(Ge);Wr=r(mo,"Tracing runs an example input tensor through your model, and captures the operations that are invoked as that input makes its way through the model\u2019s layers so that an executable or "),ts=a(mo,"CODE",{});var Um=o(ts);kr=r(Um,"ScriptFunction"),Um.forEach(l),Vr=r(mo," is returned that will be optimized using just-in-time compilation."),mo.forEach(l),ka=f(e),Nt=a(e,"P",{});var vm=o(Nt);$r=r(vm,"To trace our UNet model, we can use the following:"),vm.forEach(l),Va=f(e),w(rt.$$.fragment,e),$a=f(e),We=a(e,"P",{});var fo=o(We);Ar=r(fo,"Then we can replace the "),ls=a(fo,"CODE",{});var _m=o(ls);Nr=r(_m,"unet"),_m.forEach(l),Rr=r(fo," attribute of the pipeline with the traced model like the following"),fo.forEach(l),Aa=f(e),w(it.$$.fragment,e),Na=f(e),le=a(e,"H2",{class:!0});var co=o(le);ke=a(co,"A",{id:!0,class:!0,href:!0});var jm=o(ke);ss=a(jm,"SPAN",{});var Em=o(ss);w(pt.$$.fragment,Em),Em.forEach(l),jm.forEach(l),Sr=f(co),as=a(co,"SPAN",{});var Im=o(as);Xr=r(Im,"Memory Efficient Attention"),Im.forEach(l),co.forEach(l),Ra=f(e),$=a(e,"P",{});var Lt=o($);Yr=r(Lt,"Recent work on optimizing the bandwitdh in the attention block has generated huge speed ups and gains in GPU memory usage. The most recent being Flash Attention from @tridao: "),mt=a(Lt,"A",{href:!0,rel:!0});var gm=o(mt);Qr=r(gm,"code"),gm.forEach(l),zr=r(Lt,", "),ft=a(Lt,"A",{href:!0,rel:!0});var Zm=o(ft);xr=r(Zm,"paper"),Zm.forEach(l),Fr=r(Lt,"."),Lt.forEach(l),Sa=f(e),Rt=a(e,"P",{});var Cm=o(Rt);Hr=r(Cm,"Here are the speedups we obtain on a few Nvidia GPUs when running the inference at 512x512 with a batch size of 1 (one prompt):"),Cm.forEach(l),Xa=f(e),Ve=a(e,"TABLE",{});var ho=o(Ve);os=a(ho,"THEAD",{});var Bm=o(os);se=a(Bm,"TR",{});var Kt=o(se);ns=a(Kt,"TH",{});var Gm=o(ns);Dr=r(Gm,"GPU"),Gm.forEach(l),Pr=f(Kt),rs=a(Kt,"TH",{});var Wm=o(rs);qr=r(Wm,"Base Attention FP16"),Wm.forEach(l),Lr=f(Kt),is=a(Kt,"TH",{});var km=o(is);Kr=r(km,"Memory Efficient Attention FP16"),km.forEach(l),Kt.forEach(l),Bm.forEach(l),Or=f(ho),j=a(ho,"TBODY",{});var E=o(j);ae=a(E,"TR",{});var Ot=o(ae);ps=a(Ot,"TD",{});var Vm=o(ps);ei=r(Vm,"NVIDIA Tesla T4"),Vm.forEach(l),ti=f(Ot),ms=a(Ot,"TD",{});var $m=o(ms);li=r($m,"3.5it/s"),$m.forEach(l),si=f(Ot),fs=a(Ot,"TD",{});var Am=o(fs);ai=r(Am,"5.5it/s"),Am.forEach(l),Ot.forEach(l),oi=f(E),oe=a(E,"TR",{});var el=o(oe);cs=a(el,"TD",{});var Nm=o(cs);ni=r(Nm,"NVIDIA 3060 RTX"),Nm.forEach(l),ri=f(el),ds=a(el,"TD",{});var Rm=o(ds);ii=r(Rm,"4.6it/s"),Rm.forEach(l),pi=f(el),hs=a(el,"TD",{});var Sm=o(hs);mi=r(Sm,"7.8it/s"),Sm.forEach(l),el.forEach(l),fi=f(E),ne=a(E,"TR",{});var tl=o(ne);us=a(tl,"TD",{});var Xm=o(us);ci=r(Xm,"NVIDIA A10G"),Xm.forEach(l),di=f(tl),ys=a(tl,"TD",{});var Ym=o(ys);hi=r(Ym,"8.88it/s"),Ym.forEach(l),ui=f(tl),ws=a(tl,"TD",{});var Qm=o(ws);yi=r(Qm,"15.6it/s"),Qm.forEach(l),tl.forEach(l),wi=f(E),re=a(E,"TR",{});var ll=o(re);Ms=a(ll,"TD",{});var zm=o(Ms);Mi=r(zm,"NVIDIA RTX A6000"),zm.forEach(l),Ji=f(ll),Js=a(ll,"TD",{});var xm=o(Js);Ti=r(xm,"11.7it/s"),xm.forEach(l),bi=f(ll),Ts=a(ll,"TD",{});var Fm=o(Ts);Ui=r(Fm,"21.09it/s"),Fm.forEach(l),ll.forEach(l),vi=f(E),ie=a(E,"TR",{});var sl=o(ie);bs=a(sl,"TD",{});var Hm=o(bs);_i=r(Hm,"NVIDIA TITAN RTX"),Hm.forEach(l),ji=f(sl),Us=a(sl,"TD",{});var Dm=o(Us);Ei=r(Dm,"12.51it/s"),Dm.forEach(l),Ii=f(sl),vs=a(sl,"TD",{});var Pm=o(vs);gi=r(Pm,"18.22it/s"),Pm.forEach(l),sl.forEach(l),Zi=f(E),pe=a(E,"TR",{});var al=o(pe);_s=a(al,"TD",{});var qm=o(_s);Ci=r(qm,"A100-SXM4-40GB"),qm.forEach(l),Bi=f(al),js=a(al,"TD",{});var Lm=o(js);Gi=r(Lm,"18.6it/s"),Lm.forEach(l),Wi=f(al),Es=a(al,"TD",{});var Km=o(Es);ki=r(Km,"29.it/s"),Km.forEach(l),al.forEach(l),Vi=f(E),me=a(E,"TR",{});var ol=o(me);Is=a(ol,"TD",{});var Om=o(Is);$i=r(Om,"A100-SXM-80GB"),Om.forEach(l),Ai=f(ol),gs=a(ol,"TD",{});var ef=o(gs);Ni=r(ef,"18.7it/s"),ef.forEach(l),Ri=f(ol),Zs=a(ol,"TD",{});var tf=o(Zs);Si=r(tf,"29.5it/s"),tf.forEach(l),ol.forEach(l),E.forEach(l),ho.forEach(l),Ya=f(e),St=a(e,"P",{});var lf=o(St);Xi=r(lf,"To leverage it just make sure you have:"),lf.forEach(l),Qa=f(e),w($e.$$.fragment,e),za=f(e),A=a(e,"UL",{});var nl=o(A);Cs=a(nl,"LI",{});var sf=o(Cs);Yi=r(sf,"PyTorch > 1.12"),sf.forEach(l),Qi=f(nl),Bs=a(nl,"LI",{});var af=o(Bs);zi=r(af,"Cuda available"),af.forEach(l),xi=f(nl),Xt=a(nl,"LI",{});var Pi=o(Xt);Yt=a(Pi,"A",{href:!0});var of=o(Yt);Fi=r(of,"Installed the xformers library"),of.forEach(l),Hi=r(Pi,"."),Pi.forEach(l),nl.forEach(l),xa=f(e),w(ct.$$.fragment,e),this.h()},h(){c(d,"name","hf:doc:metadata"),c(d,"content",JSON.stringify(wf)),c(U,"id","memory-and-speed"),c(U,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(U,"href","#memory-and-speed"),c(h,"class","relative group"),c(Re,"href","https://github.com/facebookresearch/xformers"),c(Re,"rel","nofollow"),c(yt,"href","xformers"),c(de,"id","use-tf32-instead-of-fp32-on-ampere-and-later-cuda-devices"),c(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(de,"href","#use-tf32-instead-of-fp32-on-ampere-and-later-cuda-devices"),c(H,"class","relative group"),c(Xe,"href","https://huggingface.co/docs/transformers/v4.18.0/en/performance#tf32"),c(Xe,"rel","nofollow"),c(ue,"id","half-precision-weights"),c(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ue,"href","#half-precision-weights"),c(D,"class","relative group"),c(we,"id","sliced-vae-decode-for-larger-batches"),c(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(we,"href","#sliced-vae-decode-for-larger-batches"),c(P,"class","relative group"),c(Tt,"href","/docs/diffusers/v0.21.0/en/api/pipelines/stable_diffusion/img2img#diffusers.StableDiffusionImg2ImgPipeline.enable_xformers_memory_efficient_attention"),c(bt,"href","/docs/diffusers/v0.21.0/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline.enable_vae_slicing"),c(Te,"id","tiled-vae-decode-and-encode-for-large-images"),c(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Te,"href","#tiled-vae-decode-and-encode-for-large-images"),c(q,"class","relative group"),c(_t,"href","/docs/diffusers/v0.21.0/en/api/pipelines/stable_diffusion/img2img#diffusers.StableDiffusionImg2ImgPipeline.enable_xformers_memory_efficient_attention"),c(jt,"href","/docs/diffusers/v0.21.0/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline.enable_vae_tiling"),c(It,"name","sequential_offloading"),c(ve,"id","offloading-to-cpu-with-accelerate-for-memory-savings"),c(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ve,"href","#offloading-to-cpu-with-accelerate-for-memory-savings"),c(L,"class","relative group"),c(Zt,"href","/docs/diffusers/v0.21.0/en/api/pipelines/stable_diffusion/latent_upscale#diffusers.StableDiffusionLatentUpscalePipeline.enable_sequential_cpu_offload"),c(Ke,"href","https://github.com/huggingface/diffusers/issues/1934"),c(Ke,"rel","nofollow"),c(Gt,"name","model_offloading"),c(Ie,"id","model-offloading-for-fast-inference-and-memory-savings"),c(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ie,"href","#model-offloading-for-fast-inference-and-memory-savings"),c(O,"class","relative group"),c(Wt,"href","#sequential_offloading"),c(st,"href","https://huggingface.co/docs/accelerate/v0.18.0/en/package_reference/big_modeling#accelerate.hooks.remove_hook_from_module"),c(st,"rel","nofollow"),c(Ce,"id","using-channels-last-memory-format"),c(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ce,"href","#using-channels-last-memory-format"),c(ee,"class","relative group"),c(Be,"id","tracing"),c(Be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Be,"href","#tracing"),c(te,"class","relative group"),c(ke,"id","memory-efficient-attention"),c(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ke,"href","#memory-efficient-attention"),c(le,"class","relative group"),c(mt,"href","https://github.com/HazyResearch/flash-attention"),c(mt,"rel","nofollow"),c(ft,"href","https://arxiv.org/pdf/2205.14135.pdf"),c(ft,"rel","nofollow"),c(Yt,"href","xformers")},m(e,i){t(document.head,d),p(e,u,i),p(e,h,i),t(h,U),t(U,v),M(_,v,null),t(h,S),t(h,Z),t(Z,ut),p(e,Ne,i),p(e,C,i),t(C,yo),t(C,rl),t(rl,wo),t(C,Mo),t(C,Re),t(Re,Jo),t(C,To),t(C,yt),t(yt,bo),t(C,Uo),p(e,$s,i),p(e,wt,i),t(wt,vo),p(e,As,i),p(e,ce,i),t(ce,il),t(il,X),t(X,Ns),t(X,_o),t(X,pl),t(pl,jo),t(X,Eo),t(X,ml),t(ml,Io),t(ce,go),t(ce,I),t(I,Y),t(Y,fl),t(fl,Zo),t(Y,Co),t(Y,cl),t(cl,Bo),t(Y,Go),t(Y,dl),t(dl,Wo),t(I,ko),t(I,Q),t(Q,hl),t(hl,Vo),t(Q,$o),t(Q,ul),t(ul,Ao),t(Q,No),t(Q,yl),t(yl,Ro),t(I,So),t(I,z),t(z,wl),t(wl,Xo),t(z,Yo),t(z,Ml),t(Ml,Qo),t(z,zo),t(z,Jl),t(Jl,xo),t(I,Fo),t(I,x),t(x,Tl),t(Tl,Ho),t(x,Do),t(x,bl),t(bl,Po),t(x,qo),t(x,Ul),t(Ul,Lo),t(I,Ko),t(I,F),t(F,vl),t(vl,Oo),t(F,en),t(F,_l),t(_l,tn),t(F,ln),t(F,jl),t(jl,sn),p(e,Rs,i),p(e,Mt,i),t(Mt,an),p(e,Ss,i),p(e,H,i),t(H,de),t(de,El),M(Se,El,null),t(H,on),t(H,Il),t(Il,nn),p(e,Xs,i),p(e,he,i),t(he,rn),t(he,Xe),t(Xe,pn),t(he,mn),p(e,Ys,i),M(Ye,e,i),p(e,Qs,i),p(e,D,i),t(D,ue),t(ue,gl),M(Qe,gl,null),t(D,fn),t(D,Zl),t(Zl,cn),p(e,zs,i),p(e,k,i),t(k,dn),t(k,Cl),t(Cl,hn),t(k,un),t(k,Bl),t(Bl,yn),t(k,wn),p(e,xs,i),M(ze,e,i),p(e,Fs,i),M(ye,e,i),p(e,Hs,i),p(e,P,i),t(P,we),t(we,Gl),M(xe,Gl,null),t(P,Mn),t(P,Wl),t(Wl,Jn),p(e,Ds,i),p(e,Jt,i),t(Jt,Tn),p(e,Ps,i),p(e,Me,i),t(Me,bn),t(Me,Tt),t(Tt,Un),t(Me,vn),p(e,qs,i),p(e,Je,i),t(Je,_n),t(Je,bt),t(bt,jn),t(Je,En),p(e,Ls,i),M(Fe,e,i),p(e,Ks,i),p(e,Ut,i),t(Ut,In),p(e,Os,i),p(e,q,i),t(q,Te),t(Te,kl),M(He,kl,null),t(q,gn),t(q,Vl),t(Vl,Zn),p(e,ea,i),p(e,vt,i),t(vt,Cn),p(e,ta,i),p(e,be,i),t(be,Bn),t(be,_t),t(_t,Gn),t(be,Wn),p(e,la,i),p(e,Ue,i),t(Ue,kn),t(Ue,jt),t(jt,Vn),t(Ue,$n),p(e,sa,i),M(De,e,i),p(e,aa,i),p(e,Et,i),t(Et,An),p(e,oa,i),p(e,It,i),p(e,na,i),p(e,L,i),t(L,ve),t(ve,$l),M(Pe,$l,null),t(L,Nn),t(L,Al),t(Al,Rn),p(e,ra,i),p(e,gt,i),t(gt,Sn),p(e,ia,i),p(e,_e,i),t(_e,Xn),t(_e,Zt),t(Zt,Yn),t(_e,Qn),p(e,pa,i),M(qe,e,i),p(e,ma,i),p(e,Ct,i),t(Ct,zn),p(e,fa,i),p(e,je,i),t(je,xn),t(je,Nl),t(Nl,Fn),t(je,Hn),p(e,ca,i),M(Ee,e,i),p(e,da,i),p(e,Bt,i),t(Bt,Dn),p(e,ha,i),M(Le,e,i),p(e,ua,i),p(e,g,i),t(g,Rl),t(Rl,Pn),t(g,qn),t(g,Sl),t(Sl,Ln),t(g,Kn),t(g,Xl),t(Xl,On),t(g,er),t(g,Ke),t(Ke,tr),t(g,lr),p(e,ya,i),p(e,K,i),t(K,Yl),t(Yl,sr),t(K,ar),t(K,Ql),t(Ql,or),t(K,nr),p(e,wa,i),p(e,Gt,i),p(e,Ma,i),p(e,O,i),t(O,Ie),t(Ie,zl),M(Oe,zl,null),t(O,rr),t(O,xl),t(xl,ir),p(e,Ja,i),p(e,et,i),t(et,Wt),t(Wt,pr),t(et,mr),p(e,Ta,i),p(e,V,i),t(V,fr),t(V,Fl),t(Fl,cr),t(V,dr),t(V,Hl),t(Hl,hr),t(V,ur),p(e,ba,i),p(e,kt,i),t(kt,yr),p(e,Ua,i),p(e,ge,i),t(ge,wr),t(ge,Dl),t(Dl,Mr),t(ge,Jr),p(e,va,i),M(tt,e,i),p(e,_a,i),p(e,Vt,i),t(Vt,Tr),p(e,ja,i),M(lt,e,i),p(e,Ea,i),M(Ze,e,i),p(e,Ia,i),p(e,G,i),t(G,Pl),t(Pl,br),t(G,Ur),t(G,ql),t(ql,vr),t(G,_r),t(G,st),t(st,jr),t(G,Er),p(e,ga,i),p(e,ee,i),t(ee,Ce),t(Ce,Ll),M(at,Ll,null),t(ee,Ir),t(ee,Kl),t(Kl,gr),p(e,Za,i),p(e,$t,i),t($t,Zr),p(e,Ca,i),p(e,At,i),t(At,Cr),p(e,Ba,i),M(ot,e,i),p(e,Ga,i),p(e,te,i),t(te,Be),t(Be,Ol),M(nt,Ol,null),t(te,Br),t(te,es),t(es,Gr),p(e,Wa,i),p(e,Ge,i),t(Ge,Wr),t(Ge,ts),t(ts,kr),t(Ge,Vr),p(e,ka,i),p(e,Nt,i),t(Nt,$r),p(e,Va,i),M(rt,e,i),p(e,$a,i),p(e,We,i),t(We,Ar),t(We,ls),t(ls,Nr),t(We,Rr),p(e,Aa,i),M(it,e,i),p(e,Na,i),p(e,le,i),t(le,ke),t(ke,ss),M(pt,ss,null),t(le,Sr),t(le,as),t(as,Xr),p(e,Ra,i),p(e,$,i),t($,Yr),t($,mt),t(mt,Qr),t($,zr),t($,ft),t(ft,xr),t($,Fr),p(e,Sa,i),p(e,Rt,i),t(Rt,Hr),p(e,Xa,i),p(e,Ve,i),t(Ve,os),t(os,se),t(se,ns),t(ns,Dr),t(se,Pr),t(se,rs),t(rs,qr),t(se,Lr),t(se,is),t(is,Kr),t(Ve,Or),t(Ve,j),t(j,ae),t(ae,ps),t(ps,ei),t(ae,ti),t(ae,ms),t(ms,li),t(ae,si),t(ae,fs),t(fs,ai),t(j,oi),t(j,oe),t(oe,cs),t(cs,ni),t(oe,ri),t(oe,ds),t(ds,ii),t(oe,pi),t(oe,hs),t(hs,mi),t(j,fi),t(j,ne),t(ne,us),t(us,ci),t(ne,di),t(ne,ys),t(ys,hi),t(ne,ui),t(ne,ws),t(ws,yi),t(j,wi),t(j,re),t(re,Ms),t(Ms,Mi),t(re,Ji),t(re,Js),t(Js,Ti),t(re,bi),t(re,Ts),t(Ts,Ui),t(j,vi),t(j,ie),t(ie,bs),t(bs,_i),t(ie,ji),t(ie,Us),t(Us,Ei),t(ie,Ii),t(ie,vs),t(vs,gi),t(j,Zi),t(j,pe),t(pe,_s),t(_s,Ci),t(pe,Bi),t(pe,js),t(js,Gi),t(pe,Wi),t(pe,Es),t(Es,ki),t(j,Vi),t(j,me),t(me,Is),t(Is,$i),t(me,Ai),t(me,gs),t(gs,Ni),t(me,Ri),t(me,Zs),t(Zs,Si),p(e,Ya,i),p(e,St,i),t(St,Xi),p(e,Qa,i),M($e,e,i),p(e,za,i),p(e,A,i),t(A,Cs),t(Cs,Yi),t(A,Qi),t(A,Bs),t(Bs,zi),t(A,xi),t(A,Xt),t(Xt,Yt),t(Yt,Fi),t(Xt,Hi),p(e,xa,i),M(ct,e,i),Fa=!0},p(e,[i]){const dt={};i&2&&(dt.$$scope={dirty:i,ctx:e}),ye.$set(dt);const Gs={};i&2&&(Gs.$$scope={dirty:i,ctx:e}),Ee.$set(Gs);const Ws={};i&2&&(Ws.$$scope={dirty:i,ctx:e}),Ze.$set(Ws);const ks={};i&2&&(ks.$$scope={dirty:i,ctx:e}),$e.$set(ks)},i(e){Fa||(J(_.$$.fragment,e),J(Se.$$.fragment,e),J(Ye.$$.fragment,e),J(Qe.$$.fragment,e),J(ze.$$.fragment,e),J(ye.$$.fragment,e),J(xe.$$.fragment,e),J(Fe.$$.fragment,e),J(He.$$.fragment,e),J(De.$$.fragment,e),J(Pe.$$.fragment,e),J(qe.$$.fragment,e),J(Ee.$$.fragment,e),J(Le.$$.fragment,e),J(Oe.$$.fragment,e),J(tt.$$.fragment,e),J(lt.$$.fragment,e),J(Ze.$$.fragment,e),J(at.$$.fragment,e),J(ot.$$.fragment,e),J(nt.$$.fragment,e),J(rt.$$.fragment,e),J(it.$$.fragment,e),J(pt.$$.fragment,e),J($e.$$.fragment,e),J(ct.$$.fragment,e),Fa=!0)},o(e){T(_.$$.fragment,e),T(Se.$$.fragment,e),T(Ye.$$.fragment,e),T(Qe.$$.fragment,e),T(ze.$$.fragment,e),T(ye.$$.fragment,e),T(xe.$$.fragment,e),T(Fe.$$.fragment,e),T(He.$$.fragment,e),T(De.$$.fragment,e),T(Pe.$$.fragment,e),T(qe.$$.fragment,e),T(Ee.$$.fragment,e),T(Le.$$.fragment,e),T(Oe.$$.fragment,e),T(tt.$$.fragment,e),T(lt.$$.fragment,e),T(Ze.$$.fragment,e),T(at.$$.fragment,e),T(ot.$$.fragment,e),T(nt.$$.fragment,e),T(rt.$$.fragment,e),T(it.$$.fragment,e),T(pt.$$.fragment,e),T($e.$$.fragment,e),T(ct.$$.fragment,e),Fa=!1},d(e){l(d),e&&l(u),e&&l(h),b(_),e&&l(Ne),e&&l(C),e&&l($s),e&&l(wt),e&&l(As),e&&l(ce),e&&l(Rs),e&&l(Mt),e&&l(Ss),e&&l(H),b(Se),e&&l(Xs),e&&l(he),e&&l(Ys),b(Ye,e),e&&l(Qs),e&&l(D),b(Qe),e&&l(zs),e&&l(k),e&&l(xs),b(ze,e),e&&l(Fs),b(ye,e),e&&l(Hs),e&&l(P),b(xe),e&&l(Ds),e&&l(Jt),e&&l(Ps),e&&l(Me),e&&l(qs),e&&l(Je),e&&l(Ls),b(Fe,e),e&&l(Ks),e&&l(Ut),e&&l(Os),e&&l(q),b(He),e&&l(ea),e&&l(vt),e&&l(ta),e&&l(be),e&&l(la),e&&l(Ue),e&&l(sa),b(De,e),e&&l(aa),e&&l(Et),e&&l(oa),e&&l(It),e&&l(na),e&&l(L),b(Pe),e&&l(ra),e&&l(gt),e&&l(ia),e&&l(_e),e&&l(pa),b(qe,e),e&&l(ma),e&&l(Ct),e&&l(fa),e&&l(je),e&&l(ca),b(Ee,e),e&&l(da),e&&l(Bt),e&&l(ha),b(Le,e),e&&l(ua),e&&l(g),e&&l(ya),e&&l(K),e&&l(wa),e&&l(Gt),e&&l(Ma),e&&l(O),b(Oe),e&&l(Ja),e&&l(et),e&&l(Ta),e&&l(V),e&&l(ba),e&&l(kt),e&&l(Ua),e&&l(ge),e&&l(va),b(tt,e),e&&l(_a),e&&l(Vt),e&&l(ja),b(lt,e),e&&l(Ea),b(Ze,e),e&&l(Ia),e&&l(G),e&&l(ga),e&&l(ee),b(at),e&&l(Za),e&&l($t),e&&l(Ca),e&&l(At),e&&l(Ba),b(ot,e),e&&l(Ga),e&&l(te),b(nt),e&&l(Wa),e&&l(Ge),e&&l(ka),e&&l(Nt),e&&l(Va),b(rt,e),e&&l($a),e&&l(We),e&&l(Aa),b(it,e),e&&l(Na),e&&l(le),b(pt),e&&l(Ra),e&&l($),e&&l(Sa),e&&l(Rt),e&&l(Xa),e&&l(Ve),e&&l(Ya),e&&l(St),e&&l(Qa),b($e,e),e&&l(za),e&&l(A),e&&l(xa),b(ct,e)}}}const wf={local:"memory-and-speed",sections:[{local:null,sections:[{local:"use-tf32-instead-of-fp32-on-ampere-and-later-cuda-devices",title:"Use tf32 instead of fp32 (on Ampere and later CUDA devices)"}],title:null},{local:"half-precision-weights",title:"Half precision weights"},{local:"sliced-vae-decode-for-larger-batches",title:"Sliced VAE decode for larger batches"},{local:"tiled-vae-decode-and-encode-for-large-images",title:"Tiled VAE decode and encode for large images"},{local:"offloading-to-cpu-with-accelerate-for-memory-savings",title:"Offloading to CPU with accelerate for memory savings"},{local:"model-offloading-for-fast-inference-and-memory-savings",title:"Model offloading for fast inference and memory savings"},{local:"using-channels-last-memory-format",title:"Using Channels Last memory format"},{local:"tracing",title:"Tracing"},{local:"memory-efficient-attention",title:"Memory Efficient Attention"}],title:"Memory and speed"};function Mf(W){return ff(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class vf extends nf{constructor(d){super();rf(this,d,Mf,yf,pf,{})}}export{vf as default,wf as metadata};
