import{S as zs,i as Fs,s as Ys,e as o,k as p,w as m,t as i,M as Ks,c as r,d as t,m as c,a as l,x as d,h,b as u,G as s,g as n,y as v,q as g,o as b,B as T,v as Ds}from"../chunks/vendor-hf-doc-builder.js";import{T as xs}from"../chunks/Tip-hf-doc-builder.js";import{I as fe}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as gt}from"../chunks/CodeBlock-hf-doc-builder.js";function Ls(L){let y;return{c(){y=i(`Creating an inference engine is a complex topic, and the "best" solution 
will most likely depend on your problem space. Are you on CPU or GPU? Do
you want the lowest latency, the highest throughput, support for
many models, or just highly optimize 1 specific model?
There are many ways to tackle this topic, so what we are going to present is a good default
to get started which may not necessarily be the most optimal solution for you.`)},l(f){y=h(f,`Creating an inference engine is a complex topic, and the "best" solution 
will most likely depend on your problem space. Are you on CPU or GPU? Do
you want the lowest latency, the highest throughput, support for
many models, or just highly optimize 1 specific model?
There are many ways to tackle this topic, so what we are going to present is a good default
to get started which may not necessarily be the most optimal solution for you.`)},m(f,w){n(f,y,w)},d(f){f&&t(y)}}}function Os(L){let y;return{c(){y=i("Do not activate this without checking it makes sense for your load!")},l(f){y=h(f,"Do not activate this without checking it makes sense for your load!")},m(f,w){n(f,y,w)},d(f){f&&t(y)}}}function ea(L){let y,f,w,$,we,V,bt,me,Tt,qe,E,Ne,B,Mt,O,Jt,Ut,Ae,ee,kt,Se,_,jt,de,It,$t,Qe,C,Et,ve,Bt,_t,Xe,P,Ze,te,Ct,Ge,R,Ve,se,qt,Pe,W,Re,ae,Nt,We,q,At,ge,St,Qt,He,H,xe,N,ze,ne,Xt,Fe,oe,Zt,Ye,re,Gt,Ke,J,A,be,x,Vt,Te,Pt,De,U,S,Me,z,Rt,Je,Wt,Le,le,Ht,Oe,Q,xt,Ue,zt,Ft,et,k,X,ke,F,Yt,je,Kt,tt,ie,Dt,st,he,Lt,at,j,Z,Ie,Y,Ot,$e,es,nt,pe,ts,ot,ce,ss,rt,I,G,Ee,K,as,Be,ns,lt,M,os,ue,rs,ls,_e,is,hs,it;return V=new fe({}),E=new xs({props:{$$slots:{default:[Ls]},$$scope:{ctx:L}}}),P=new gt({props:{code:"ZnJvbSUyMHN0YXJsZXR0ZS5hcHBsaWNhdGlvbnMlMjBpbXBvcnQlMjBTdGFybGV0dGUlMEFmcm9tJTIwc3RhcmxldHRlLnJlc3BvbnNlcyUyMGltcG9ydCUyMEpTT05SZXNwb25zZSUwQWZyb20lMjBzdGFybGV0dGUucm91dGluZyUyMGltcG9ydCUyMFJvdXRlJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBaW1wb3J0JTIwYXN5bmNpbyUwQSUwQSUwQWFzeW5jJTIwZGVmJTIwaG9tZXBhZ2UocmVxdWVzdCklM0ElMEElMjAlMjAlMjAlMjBwYXlsb2FkJTIwJTNEJTIwYXdhaXQlMjByZXF1ZXN0LmJvZHkoKSUwQSUyMCUyMCUyMCUyMHN0cmluZyUyMCUzRCUyMHBheWxvYWQuZGVjb2RlKCUyMnV0Zi04JTIyKSUwQSUyMCUyMCUyMCUyMHJlc3BvbnNlX3ElMjAlM0QlMjBhc3luY2lvLlF1ZXVlKCklMEElMjAlMjAlMjAlMjBhd2FpdCUyMHJlcXVlc3QuYXBwLm1vZGVsX3F1ZXVlLnB1dCgoc3RyaW5nJTJDJTIwcmVzcG9uc2VfcSkpJTBBJTIwJTIwJTIwJTIwb3V0cHV0JTIwJTNEJTIwYXdhaXQlMjByZXNwb25zZV9xLmdldCgpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwSlNPTlJlc3BvbnNlKG91dHB1dCklMEElMEElMEFhc3luYyUyMGRlZiUyMHNlcnZlcl9sb29wKHEpJTNBJTBBJTIwJTIwJTIwJTIwcGlwZSUyMCUzRCUyMHBpcGVsaW5lKG1vZGVsJTNEJTIyYmVydC1iYXNlLXVuY2FzZWQlMjIpJTBBJTIwJTIwJTIwJTIwd2hpbGUlMjBUcnVlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKHN0cmluZyUyQyUyMHJlc3BvbnNlX3EpJTIwJTNEJTIwYXdhaXQlMjBxLmdldCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0JTIwJTNEJTIwcGlwZShzdHJpbmcpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYXdhaXQlMjByZXNwb25zZV9xLnB1dChvdXQpJTBBJTBBJTBBYXBwJTIwJTNEJTIwU3RhcmxldHRlKCUwQSUyMCUyMCUyMCUyMHJvdXRlcyUzRCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFJvdXRlKCUyMiUyRiUyMiUyQyUyMGhvbWVwYWdlJTJDJTIwbWV0aG9kcyUzRCU1QiUyMlBPU1QlMjIlNUQpJTJDJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBKSUwQSUwQSUwQSU0MGFwcC5vbl9ldmVudCglMjJzdGFydHVwJTIyKSUwQWFzeW5jJTIwZGVmJTIwc3RhcnR1cF9ldmVudCgpJTNBJTBBJTIwJTIwJTIwJTIwcSUyMCUzRCUyMGFzeW5jaW8uUXVldWUoKSUwQSUyMCUyMCUyMCUyMGFwcC5tb2RlbF9xdWV1ZSUyMCUzRCUyMHElMEElMjAlMjAlMjAlMjBhc3luY2lvLmNyZWF0ZV90YXNrKHNlcnZlcl9sb29wKHEpKQ==",highlighted:`<span class="hljs-keyword">from</span> starlette.applications <span class="hljs-keyword">import</span> Starlette
<span class="hljs-keyword">from</span> starlette.responses <span class="hljs-keyword">import</span> JSONResponse
<span class="hljs-keyword">from</span> starlette.routing <span class="hljs-keyword">import</span> Route
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-keyword">import</span> asyncio


<span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">homepage</span>(<span class="hljs-params">request</span>):
    payload = <span class="hljs-keyword">await</span> request.body()
    string = payload.decode(<span class="hljs-string">&quot;utf-8&quot;</span>)
    response_q = asyncio.Queue()
    <span class="hljs-keyword">await</span> request.app.model_queue.put((string, response_q))
    output = <span class="hljs-keyword">await</span> response_q.get()
    <span class="hljs-keyword">return</span> JSONResponse(output)


<span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">server_loop</span>(<span class="hljs-params">q</span>):
    pipe = pipeline(model=<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
    <span class="hljs-keyword">while</span> <span class="hljs-literal">True</span>:
        (string, response_q) = <span class="hljs-keyword">await</span> q.get()
        out = pipe(string)
        <span class="hljs-keyword">await</span> response_q.put(out)


app = Starlette(
    routes=[
        Route(<span class="hljs-string">&quot;/&quot;</span>, homepage, methods=[<span class="hljs-string">&quot;POST&quot;</span>]),
    ],
)


<span class="hljs-meta">@app.on_event(<span class="hljs-params"><span class="hljs-string">&quot;startup&quot;</span></span>)</span>
<span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">startup_event</span>():
    q = asyncio.Queue()
    app.model_queue = q
    asyncio.create_task(server_loop(q))`}}),R=new gt({props:{code:"dXZpY29ybiUyMHNlcnZlciUzQWFwcA==",highlighted:"uvicorn server:app"}}),W=new gt({props:{code:"Y3VybCUyMC1YJTIwUE9TVCUyMC1kJTIwJTIydGVzdCUyMCU1Qk1BU0slNUQlMjIlMjBodHRwJTNBJTJGJTJGbG9jYWxob3N0JTNBODAwMCUyRiUwQSUyMyU1QiU3QiUyMnNjb3JlJTIyJTNBMC43NzQyOTM2MDE1MTI5MDg5JTJDJTIydG9rZW4lMjIlM0ExMDEyJTJDJTIydG9rZW5fc3RyJTIyJTNBJTIyLiUyMiUyQyUyMnNlcXVlbmNlJTIyJTNBJTIydGVzdC4lMjIlN0QlMkMuLi4lNUQ=",highlighted:`curl -X POST -d <span class="hljs-string">&quot;test [MASK]&quot;</span> http://localhost:8000/
<span class="hljs-comment">#[{&quot;score&quot;:0.7742936015129089,&quot;token&quot;:1012,&quot;token_str&quot;:&quot;.&quot;,&quot;sequence&quot;:&quot;test.&quot;},...]</span>`}}),H=new gt({props:{code:"KHN0cmluZyUyQyUyMHJxKSUyMCUzRCUyMGF3YWl0JTIwcS5nZXQoKSUwQXN0cmluZ3MlMjAlM0QlMjAlNUIlNUQlMEFxdWV1ZXMlMjAlM0QlMjAlNUIlNUQlMEF3aGlsZSUyMFRydWUlM0ElMEElMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAoc3RyaW5nJTJDJTIwcnEpJTIwJTNEJTIwYXdhaXQlMjBhc3luY2lvLndhaXRfZm9yKHEuZ2V0KCklMkMlMjB0aW1lb3V0JTNEMC4wMDEpJTIwJTIwJTIzJTIwMW1zJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwYXN5bmNpby5leGNlcHRpb25zLlRpbWVvdXRFcnJvciUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGJyZWFrJTBBJTIwJTIwJTIwJTIwc3RyaW5ncy5hcHBlbmQoc3RyaW5nKSUwQSUyMCUyMCUyMCUyMHF1ZXVlcy5hcHBlbmQocnEpJTBBc3RyaW5ncyUwQW91dHMlMjAlM0QlMjBwaXBlKHN0cmluZ3MlMkMlMjBiYXRjaF9zaXplJTNEbGVuKHN0cmluZ3MpKSUwQWZvciUyMHJxJTJDJTIwb3V0JTIwaW4lMjB6aXAocXVldWVzJTJDJTIwb3V0cyklM0ElMEElMjAlMjAlMjAlMjBhd2FpdCUyMHJxLnB1dChvdXQp",highlighted:`(string, rq) = <span class="hljs-keyword">await</span> q.get()
strings = []
queues = []
<span class="hljs-keyword">while</span> <span class="hljs-literal">True</span>:
    <span class="hljs-keyword">try</span>:
        (string, rq) = <span class="hljs-keyword">await</span> asyncio.wait_for(q.get(), timeout=<span class="hljs-number">0.001</span>)  <span class="hljs-comment"># 1ms</span>
    <span class="hljs-keyword">except</span> asyncio.exceptions.TimeoutError:
        <span class="hljs-keyword">break</span>
    strings.append(string)
    queues.append(rq)
strings
outs = pipe(strings, batch_size=<span class="hljs-built_in">len</span>(strings))
<span class="hljs-keyword">for</span> rq, out <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(queues, outs):
    <span class="hljs-keyword">await</span> rq.put(out)`}}),N=new xs({props:{warning:!0,$$slots:{default:[Os]},$$scope:{ctx:L}}}),x=new fe({}),z=new fe({}),F=new fe({}),Y=new fe({}),K=new fe({}),{c(){y=o("meta"),f=p(),w=o("h1"),$=o("a"),we=o("span"),m(V.$$.fragment),bt=p(),me=o("span"),Tt=i("Using pipelines for a webserver"),qe=p(),m(E.$$.fragment),Ne=p(),B=o("p"),Mt=i("The key thing to understand is that we can use an iterator, just like you would "),O=o("a"),Jt=i(`on a
dataset`),Ut=i(`, since a webserver is basically a system that waits for requests and
treats them as they come in.`),Ae=p(),ee=o("p"),kt=i(`Usually webservers are multiplexed (multithreaded, async, etc..) to handle various
requests concurrently. Pipelines on the other hand (and mostly the underlying models)
are not really great for parallelism; they take up a lot of RAM, so it\u2019s best to give them all the available resources when they are running or it\u2019s a compute-intensive job.`),Se=p(),_=o("p"),jt=i(`We are going to solve that by having the webserver handle the light load of receiving
and sending requests, and having a single thread handling the actual work.
This example is going to use `),de=o("code"),It=i("starlette"),$t=i(`. The actual framework is not really
important, but you might have to tune or change the code if you are using another
one to achieve the same effect.`),Qe=p(),C=o("p"),Et=i("Create "),ve=o("code"),Bt=i("server.py"),_t=i(":"),Xe=p(),m(P.$$.fragment),Ze=p(),te=o("p"),Ct=i("Now you can start it with:"),Ge=p(),m(R.$$.fragment),Ve=p(),se=o("p"),qt=i("And you can query it:"),Pe=p(),m(W.$$.fragment),Re=p(),ae=o("p"),Nt=i("And there you go, now you have a good idea of how to create a webserver!"),We=p(),q=o("p"),At=i("What is really important is that we load the model only "),ge=o("strong"),St=i("once"),Qt=i(`, so there are no copies
of the model on the webserver. This way, no unnecessary RAM is being used.
Then the queuing mechanism allows you to do fancy stuff like maybe accumulating a few
items before inferring to use dynamic batching:`),He=p(),m(H.$$.fragment),xe=p(),m(N.$$.fragment),ze=p(),ne=o("p"),Xt=i(`The proposed code is optimized for readability, not for being the best code.
First of all, there\u2019s no batch size limit which is usually not a
great idea. Next, the timeout is reset on every queue fetch, meaning you could
wait much more than 1ms before running the inference (delaying the first request
by that much).`),Fe=p(),oe=o("p"),Zt=i("It would be better to have a single 1ms deadline."),Ye=p(),re=o("p"),Gt=i(`This will always wait for 1ms even if the queue is empty, which might not be the
best since you probably want to start doing inference if there\u2019s nothing in the queue.
But maybe it does make sense if batching is really crucial for your use case.
Again, there\u2019s really no one best solution.`),Ke=p(),J=o("h2"),A=o("a"),be=o("span"),m(x.$$.fragment),Vt=p(),Te=o("span"),Pt=i("Few things you might want to consider"),De=p(),U=o("h3"),S=o("a"),Me=o("span"),m(z.$$.fragment),Rt=p(),Je=o("span"),Wt=i("Error checking"),Le=p(),le=o("p"),Ht=i(`There\u2019s a lot that can go wrong in production: out of memory, out of space,
loading the model might fail, the query might be wrong, the query might be
correct but still fail to run because of a model misconfiguration, and so on.`),Oe=p(),Q=o("p"),xt=i(`Generally, it\u2019s good if the server outputs the errors to the user, so
adding a lot of `),Ue=o("code"),zt=i("try..except"),Ft=i(` statements to show those errors is a good
idea. But keep in mind it may also be a security risk to reveal all those errors depending
on your security context.`),et=p(),k=o("h3"),X=o("a"),ke=o("span"),m(F.$$.fragment),Yt=p(),je=o("span"),Kt=i("Circuit breaking"),tt=p(),ie=o("p"),Dt=i(`Webservers usually look better when they do circuit breaking. It means they
return proper errors when they\u2019re overloaded instead of just waiting for the query indefinitely. Return a 503 error instead of waiting for a super long time or a 504 after a long time.`),st=p(),he=o("p"),Lt=i(`This is relatively easy to implement in the proposed code since there is a single queue.
Looking at the queue size is a basic way to start returning errors before your
webserver fails under load.`),at=p(),j=o("h3"),Z=o("a"),Ie=o("span"),m(Y.$$.fragment),Ot=p(),$e=o("span"),es=i("Blocking the main thread"),nt=p(),pe=o("p"),ts=i(`Currently PyTorch is not async aware, and computation will block the main
thread while running. That means it would be better if PyTorch was forced to run
on its own thread/process. This wasn\u2019t done here because the code is a lot more
complex (mostly because threads and async and queues don\u2019t play nice together).
But ultimately it does the same thing.`),ot=p(),ce=o("p"),ss=i(`This would be important if the inference of single items were long (> 1s) because
in this case, it means every query during inference would have to wait for 1s before
even receiving an error.`),rt=p(),I=o("h3"),G=o("a"),Ee=o("span"),m(K.$$.fragment),as=p(),Be=o("span"),ns=i("Dynamic batching"),lt=p(),M=o("p"),os=i(`In general, batching is not necessarily an improvement over passing 1 item at
a time (see `),ue=o("a"),rs=i("batching details"),ls=i(` for more information). But it can be very effective
when used in the correct setting. In the API, there is no dynamic
batching by default (too much opportunity for a slowdown). But for BLOOM inference -
which is a very large model - dynamic batching is `),_e=o("strong"),is=i("essential"),hs=i(" to provide a decent experience for everyone."),this.h()},l(e){const a=Ks('[data-svelte="svelte-1phssyn"]',document.head);y=r(a,"META",{name:!0,content:!0}),a.forEach(t),f=c(e),w=r(e,"H1",{class:!0});var D=l(w);$=r(D,"A",{id:!0,class:!0,href:!0});var Ce=l($);we=r(Ce,"SPAN",{});var ps=l(we);d(V.$$.fragment,ps),ps.forEach(t),Ce.forEach(t),bt=c(D),me=r(D,"SPAN",{});var cs=l(me);Tt=h(cs,"Using pipelines for a webserver"),cs.forEach(t),D.forEach(t),qe=c(e),d(E.$$.fragment,e),Ne=c(e),B=r(e,"P",{});var ht=l(B);Mt=h(ht,"The key thing to understand is that we can use an iterator, just like you would "),O=r(ht,"A",{href:!0});var us=l(O);Jt=h(us,`on a
dataset`),us.forEach(t),Ut=h(ht,`, since a webserver is basically a system that waits for requests and
treats them as they come in.`),ht.forEach(t),Ae=c(e),ee=r(e,"P",{});var ys=l(ee);kt=h(ys,`Usually webservers are multiplexed (multithreaded, async, etc..) to handle various
requests concurrently. Pipelines on the other hand (and mostly the underlying models)
are not really great for parallelism; they take up a lot of RAM, so it\u2019s best to give them all the available resources when they are running or it\u2019s a compute-intensive job.`),ys.forEach(t),Se=c(e),_=r(e,"P",{});var pt=l(_);jt=h(pt,`We are going to solve that by having the webserver handle the light load of receiving
and sending requests, and having a single thread handling the actual work.
This example is going to use `),de=r(pt,"CODE",{});var fs=l(de);It=h(fs,"starlette"),fs.forEach(t),$t=h(pt,`. The actual framework is not really
important, but you might have to tune or change the code if you are using another
one to achieve the same effect.`),pt.forEach(t),Qe=c(e),C=r(e,"P",{});var ct=l(C);Et=h(ct,"Create "),ve=r(ct,"CODE",{});var ws=l(ve);Bt=h(ws,"server.py"),ws.forEach(t),_t=h(ct,":"),ct.forEach(t),Xe=c(e),d(P.$$.fragment,e),Ze=c(e),te=r(e,"P",{});var ms=l(te);Ct=h(ms,"Now you can start it with:"),ms.forEach(t),Ge=c(e),d(R.$$.fragment,e),Ve=c(e),se=r(e,"P",{});var ds=l(se);qt=h(ds,"And you can query it:"),ds.forEach(t),Pe=c(e),d(W.$$.fragment,e),Re=c(e),ae=r(e,"P",{});var vs=l(ae);Nt=h(vs,"And there you go, now you have a good idea of how to create a webserver!"),vs.forEach(t),We=c(e),q=r(e,"P",{});var ut=l(q);At=h(ut,"What is really important is that we load the model only "),ge=r(ut,"STRONG",{});var gs=l(ge);St=h(gs,"once"),gs.forEach(t),Qt=h(ut,`, so there are no copies
of the model on the webserver. This way, no unnecessary RAM is being used.
Then the queuing mechanism allows you to do fancy stuff like maybe accumulating a few
items before inferring to use dynamic batching:`),ut.forEach(t),He=c(e),d(H.$$.fragment,e),xe=c(e),d(N.$$.fragment,e),ze=c(e),ne=r(e,"P",{});var bs=l(ne);Xt=h(bs,`The proposed code is optimized for readability, not for being the best code.
First of all, there\u2019s no batch size limit which is usually not a
great idea. Next, the timeout is reset on every queue fetch, meaning you could
wait much more than 1ms before running the inference (delaying the first request
by that much).`),bs.forEach(t),Fe=c(e),oe=r(e,"P",{});var Ts=l(oe);Zt=h(Ts,"It would be better to have a single 1ms deadline."),Ts.forEach(t),Ye=c(e),re=r(e,"P",{});var Ms=l(re);Gt=h(Ms,`This will always wait for 1ms even if the queue is empty, which might not be the
best since you probably want to start doing inference if there\u2019s nothing in the queue.
But maybe it does make sense if batching is really crucial for your use case.
Again, there\u2019s really no one best solution.`),Ms.forEach(t),Ke=c(e),J=r(e,"H2",{class:!0});var yt=l(J);A=r(yt,"A",{id:!0,class:!0,href:!0});var Js=l(A);be=r(Js,"SPAN",{});var Us=l(be);d(x.$$.fragment,Us),Us.forEach(t),Js.forEach(t),Vt=c(yt),Te=r(yt,"SPAN",{});var ks=l(Te);Pt=h(ks,"Few things you might want to consider"),ks.forEach(t),yt.forEach(t),De=c(e),U=r(e,"H3",{class:!0});var ft=l(U);S=r(ft,"A",{id:!0,class:!0,href:!0});var js=l(S);Me=r(js,"SPAN",{});var Is=l(Me);d(z.$$.fragment,Is),Is.forEach(t),js.forEach(t),Rt=c(ft),Je=r(ft,"SPAN",{});var $s=l(Je);Wt=h($s,"Error checking"),$s.forEach(t),ft.forEach(t),Le=c(e),le=r(e,"P",{});var Es=l(le);Ht=h(Es,`There\u2019s a lot that can go wrong in production: out of memory, out of space,
loading the model might fail, the query might be wrong, the query might be
correct but still fail to run because of a model misconfiguration, and so on.`),Es.forEach(t),Oe=c(e),Q=r(e,"P",{});var wt=l(Q);xt=h(wt,`Generally, it\u2019s good if the server outputs the errors to the user, so
adding a lot of `),Ue=r(wt,"CODE",{});var Bs=l(Ue);zt=h(Bs,"try..except"),Bs.forEach(t),Ft=h(wt,` statements to show those errors is a good
idea. But keep in mind it may also be a security risk to reveal all those errors depending
on your security context.`),wt.forEach(t),et=c(e),k=r(e,"H3",{class:!0});var mt=l(k);X=r(mt,"A",{id:!0,class:!0,href:!0});var _s=l(X);ke=r(_s,"SPAN",{});var Cs=l(ke);d(F.$$.fragment,Cs),Cs.forEach(t),_s.forEach(t),Yt=c(mt),je=r(mt,"SPAN",{});var qs=l(je);Kt=h(qs,"Circuit breaking"),qs.forEach(t),mt.forEach(t),tt=c(e),ie=r(e,"P",{});var Ns=l(ie);Dt=h(Ns,`Webservers usually look better when they do circuit breaking. It means they
return proper errors when they\u2019re overloaded instead of just waiting for the query indefinitely. Return a 503 error instead of waiting for a super long time or a 504 after a long time.`),Ns.forEach(t),st=c(e),he=r(e,"P",{});var As=l(he);Lt=h(As,`This is relatively easy to implement in the proposed code since there is a single queue.
Looking at the queue size is a basic way to start returning errors before your
webserver fails under load.`),As.forEach(t),at=c(e),j=r(e,"H3",{class:!0});var dt=l(j);Z=r(dt,"A",{id:!0,class:!0,href:!0});var Ss=l(Z);Ie=r(Ss,"SPAN",{});var Qs=l(Ie);d(Y.$$.fragment,Qs),Qs.forEach(t),Ss.forEach(t),Ot=c(dt),$e=r(dt,"SPAN",{});var Xs=l($e);es=h(Xs,"Blocking the main thread"),Xs.forEach(t),dt.forEach(t),nt=c(e),pe=r(e,"P",{});var Zs=l(pe);ts=h(Zs,`Currently PyTorch is not async aware, and computation will block the main
thread while running. That means it would be better if PyTorch was forced to run
on its own thread/process. This wasn\u2019t done here because the code is a lot more
complex (mostly because threads and async and queues don\u2019t play nice together).
But ultimately it does the same thing.`),Zs.forEach(t),ot=c(e),ce=r(e,"P",{});var Gs=l(ce);ss=h(Gs,`This would be important if the inference of single items were long (> 1s) because
in this case, it means every query during inference would have to wait for 1s before
even receiving an error.`),Gs.forEach(t),rt=c(e),I=r(e,"H3",{class:!0});var vt=l(I);G=r(vt,"A",{id:!0,class:!0,href:!0});var Vs=l(G);Ee=r(Vs,"SPAN",{});var Ps=l(Ee);d(K.$$.fragment,Ps),Ps.forEach(t),Vs.forEach(t),as=c(vt),Be=r(vt,"SPAN",{});var Rs=l(Be);ns=h(Rs,"Dynamic batching"),Rs.forEach(t),vt.forEach(t),lt=c(e),M=r(e,"P",{});var ye=l(M);os=h(ye,`In general, batching is not necessarily an improvement over passing 1 item at
a time (see `),ue=r(ye,"A",{href:!0});var Ws=l(ue);rs=h(Ws,"batching details"),Ws.forEach(t),ls=h(ye,` for more information). But it can be very effective
when used in the correct setting. In the API, there is no dynamic
batching by default (too much opportunity for a slowdown). But for BLOOM inference -
which is a very large model - dynamic batching is `),_e=r(ye,"STRONG",{});var Hs=l(_e);is=h(Hs,"essential"),Hs.forEach(t),hs=h(ye," to provide a decent experience for everyone."),ye.forEach(t),this.h()},h(){u(y,"name","hf:doc:metadata"),u(y,"content",JSON.stringify(ta)),u($,"id","using-pipelines-for-a-webserver"),u($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u($,"href","#using-pipelines-for-a-webserver"),u(w,"class","relative group"),u(O,"href","pipeline_tutorial#using-pipelines-on-a-dataset"),u(A,"id","few-things-you-might-want-to-consider"),u(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(A,"href","#few-things-you-might-want-to-consider"),u(J,"class","relative group"),u(S,"id","error-checking"),u(S,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(S,"href","#error-checking"),u(U,"class","relative group"),u(X,"id","circuit-breaking"),u(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(X,"href","#circuit-breaking"),u(k,"class","relative group"),u(Z,"id","blocking-the-main-thread"),u(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Z,"href","#blocking-the-main-thread"),u(j,"class","relative group"),u(G,"id","dynamic-batching"),u(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(G,"href","#dynamic-batching"),u(I,"class","relative group"),u(ue,"href","./main_classes/pipelines#pipeline-batching")},m(e,a){s(document.head,y),n(e,f,a),n(e,w,a),s(w,$),s($,we),v(V,we,null),s(w,bt),s(w,me),s(me,Tt),n(e,qe,a),v(E,e,a),n(e,Ne,a),n(e,B,a),s(B,Mt),s(B,O),s(O,Jt),s(B,Ut),n(e,Ae,a),n(e,ee,a),s(ee,kt),n(e,Se,a),n(e,_,a),s(_,jt),s(_,de),s(de,It),s(_,$t),n(e,Qe,a),n(e,C,a),s(C,Et),s(C,ve),s(ve,Bt),s(C,_t),n(e,Xe,a),v(P,e,a),n(e,Ze,a),n(e,te,a),s(te,Ct),n(e,Ge,a),v(R,e,a),n(e,Ve,a),n(e,se,a),s(se,qt),n(e,Pe,a),v(W,e,a),n(e,Re,a),n(e,ae,a),s(ae,Nt),n(e,We,a),n(e,q,a),s(q,At),s(q,ge),s(ge,St),s(q,Qt),n(e,He,a),v(H,e,a),n(e,xe,a),v(N,e,a),n(e,ze,a),n(e,ne,a),s(ne,Xt),n(e,Fe,a),n(e,oe,a),s(oe,Zt),n(e,Ye,a),n(e,re,a),s(re,Gt),n(e,Ke,a),n(e,J,a),s(J,A),s(A,be),v(x,be,null),s(J,Vt),s(J,Te),s(Te,Pt),n(e,De,a),n(e,U,a),s(U,S),s(S,Me),v(z,Me,null),s(U,Rt),s(U,Je),s(Je,Wt),n(e,Le,a),n(e,le,a),s(le,Ht),n(e,Oe,a),n(e,Q,a),s(Q,xt),s(Q,Ue),s(Ue,zt),s(Q,Ft),n(e,et,a),n(e,k,a),s(k,X),s(X,ke),v(F,ke,null),s(k,Yt),s(k,je),s(je,Kt),n(e,tt,a),n(e,ie,a),s(ie,Dt),n(e,st,a),n(e,he,a),s(he,Lt),n(e,at,a),n(e,j,a),s(j,Z),s(Z,Ie),v(Y,Ie,null),s(j,Ot),s(j,$e),s($e,es),n(e,nt,a),n(e,pe,a),s(pe,ts),n(e,ot,a),n(e,ce,a),s(ce,ss),n(e,rt,a),n(e,I,a),s(I,G),s(G,Ee),v(K,Ee,null),s(I,as),s(I,Be),s(Be,ns),n(e,lt,a),n(e,M,a),s(M,os),s(M,ue),s(ue,rs),s(M,ls),s(M,_e),s(_e,is),s(M,hs),it=!0},p(e,[a]){const D={};a&2&&(D.$$scope={dirty:a,ctx:e}),E.$set(D);const Ce={};a&2&&(Ce.$$scope={dirty:a,ctx:e}),N.$set(Ce)},i(e){it||(g(V.$$.fragment,e),g(E.$$.fragment,e),g(P.$$.fragment,e),g(R.$$.fragment,e),g(W.$$.fragment,e),g(H.$$.fragment,e),g(N.$$.fragment,e),g(x.$$.fragment,e),g(z.$$.fragment,e),g(F.$$.fragment,e),g(Y.$$.fragment,e),g(K.$$.fragment,e),it=!0)},o(e){b(V.$$.fragment,e),b(E.$$.fragment,e),b(P.$$.fragment,e),b(R.$$.fragment,e),b(W.$$.fragment,e),b(H.$$.fragment,e),b(N.$$.fragment,e),b(x.$$.fragment,e),b(z.$$.fragment,e),b(F.$$.fragment,e),b(Y.$$.fragment,e),b(K.$$.fragment,e),it=!1},d(e){t(y),e&&t(f),e&&t(w),T(V),e&&t(qe),T(E,e),e&&t(Ne),e&&t(B),e&&t(Ae),e&&t(ee),e&&t(Se),e&&t(_),e&&t(Qe),e&&t(C),e&&t(Xe),T(P,e),e&&t(Ze),e&&t(te),e&&t(Ge),T(R,e),e&&t(Ve),e&&t(se),e&&t(Pe),T(W,e),e&&t(Re),e&&t(ae),e&&t(We),e&&t(q),e&&t(He),T(H,e),e&&t(xe),T(N,e),e&&t(ze),e&&t(ne),e&&t(Fe),e&&t(oe),e&&t(Ye),e&&t(re),e&&t(Ke),e&&t(J),T(x),e&&t(De),e&&t(U),T(z),e&&t(Le),e&&t(le),e&&t(Oe),e&&t(Q),e&&t(et),e&&t(k),T(F),e&&t(tt),e&&t(ie),e&&t(st),e&&t(he),e&&t(at),e&&t(j),T(Y),e&&t(nt),e&&t(pe),e&&t(ot),e&&t(ce),e&&t(rt),e&&t(I),T(K),e&&t(lt),e&&t(M)}}}const ta={local:"using-pipelines-for-a-webserver",sections:[{local:"few-things-you-might-want-to-consider",sections:[{local:"error-checking",title:"Error checking"},{local:"circuit-breaking",title:"Circuit breaking"},{local:"blocking-the-main-thread",title:"Blocking the main thread"},{local:"dynamic-batching",title:"Dynamic batching"}],title:"Few things you might want to consider"}],title:"Using pipelines for a webserver"};function sa(L){return Ds(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class la extends zs{constructor(y){super();Fs(this,y,sa,ea,Ys,{})}}export{la as default,ta as metadata};
