import{S as Mc,i as Tc,s as $c,e as s,k as d,w as y,t as o,M as Cc,c as a,d as n,m as p,a as i,x as I,h as r,b as l,N as kc,G as e,g as u,y as B,q as w,o as M,B as T,v as Ec,L as or}from"../../chunks/vendor-hf-doc-builder.js";import{T as wc}from"../../chunks/Tip-hf-doc-builder.js";import{D as U}from"../../chunks/Docstring-hf-doc-builder.js";import{C as rr}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as ue}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as nr}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function jc(x){let f,C,_,h,v;return h=new rr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMCglMEElMjAlMjAlMjAlMjBJbnN0cnVjdEJsaXBWaXNpb25Db25maWclMkMlMEElMjAlMjAlMjAlMjBJbnN0cnVjdEJsaXBRRm9ybWVyQ29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwT1BUQ29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwSW5zdHJ1Y3RCbGlwQ29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwSW5zdHJ1Y3RCbGlwRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTJDJTBBKSUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBJbnN0cnVjdEJsaXBDb25maWclMjB3aXRoJTIwU2FsZXNmb3JjZSUyRmluc3RydWN0LWJsaXAtZmxhbi10NSUyMHN0eWxlJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBJbnN0cnVjdEJsaXBDb25maWcoKSUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBJbnN0cnVjdEJsaXBGb3JDb25kaXRpb25hbEdlbmVyYXRpb24lMjAod2l0aCUyMHJhbmRvbSUyMHdlaWdodHMpJTIwZnJvbSUyMHRoZSUyMFNhbGVzZm9yY2UlMkZpbnN0cnVjdC1ibGlwLWZsYW4tdDUlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFtb2RlbCUyMCUzRCUyMEluc3RydWN0QmxpcEZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbihjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWclMEElMEElMjMlMjBXZSUyMGNhbiUyMGFsc28lMjBpbml0aWFsaXplJTIwYSUyMEluc3RydWN0QmxpcENvbmZpZyUyMGZyb20lMjBhJTIwSW5zdHJ1Y3RCbGlwVmlzaW9uQ29uZmlnJTJDJTIwSW5zdHJ1Y3RCbGlwUUZvcm1lckNvbmZpZyUyMGFuZCUyMGFueSUyMFByZXRyYWluZWRDb25maWclMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBJbnN0cnVjdEJMSVAlMjB2aXNpb24lMkMlMjBJbnN0cnVjdEJMSVAlMjBRLUZvcm1lciUyMGFuZCUyMGxhbmd1YWdlJTIwbW9kZWwlMjBjb25maWd1cmF0aW9ucyUwQXZpc2lvbl9jb25maWclMjAlM0QlMjBJbnN0cnVjdEJsaXBWaXNpb25Db25maWcoKSUwQXFmb3JtZXJfY29uZmlnJTIwJTNEJTIwSW5zdHJ1Y3RCbGlwUUZvcm1lckNvbmZpZygpJTBBdGV4dF9jb25maWclMjAlM0QlMjBPUFRDb25maWcoKSUwQSUwQWNvbmZpZyUyMCUzRCUyMEluc3RydWN0QmxpcENvbmZpZy5mcm9tX3RleHRfdmlzaW9uX2NvbmZpZ3ModmlzaW9uX2NvbmZpZyUyQyUyMHFmb3JtZXJfY29uZmlnJTJDJTIwdGV4dF9jb25maWcp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> (
<span class="hljs-meta">... </span>    InstructBlipVisionConfig,
<span class="hljs-meta">... </span>    InstructBlipQFormerConfig,
<span class="hljs-meta">... </span>    OPTConfig,
<span class="hljs-meta">... </span>    InstructBlipConfig,
<span class="hljs-meta">... </span>    InstructBlipForConditionalGeneration,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipConfig with Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = InstructBlipConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipForConditionalGeneration (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipForConditionalGeneration(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># We can also initialize a InstructBlipConfig from a InstructBlipVisionConfig, InstructBlipQFormerConfig and any PretrainedConfig</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing InstructBLIP vision, InstructBLIP Q-Former and language model configurations</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>vision_config = InstructBlipVisionConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>qformer_config = InstructBlipQFormerConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>text_config = OPTConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span>config = InstructBlipConfig.from_text_vision_configs(vision_config, qformer_config, text_config)`}}),{c(){f=s("p"),C=o("Example:"),_=d(),y(h.$$.fragment)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Example:"),g.forEach(n),_=p(c),I(h.$$.fragment,c)},m(c,g){u(c,f,g),e(f,C),u(c,_,g),B(h,c,g),v=!0},p:or,i(c){v||(w(h.$$.fragment,c),v=!0)},o(c){M(h.$$.fragment,c),v=!1},d(c){c&&n(f),c&&n(_),T(h,c)}}}function Fc(x){let f,C,_,h,v;return h=new rr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEluc3RydWN0QmxpcFZpc2lvbkNvbmZpZyUyQyUyMEluc3RydWN0QmxpcFZpc2lvbk1vZGVsJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEluc3RydWN0QmxpcFZpc2lvbkNvbmZpZyUyMHdpdGglMjBTYWxlc2ZvcmNlJTJGaW5zdHJ1Y3QtYmxpcC1mbGFuLXQ1JTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMEluc3RydWN0QmxpcFZpc2lvbkNvbmZpZygpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEluc3RydWN0QmxpcFZpc2lvbk1vZGVsJTIwKHdpdGglMjByYW5kb20lMjB3ZWlnaHRzKSUyMGZyb20lMjB0aGUlMjBTYWxlc2ZvcmNlJTJGaW5zdHJ1Y3QtYmxpcC1mbGFuLXQ1JTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBbW9kZWwlMjAlM0QlMjBJbnN0cnVjdEJsaXBWaXNpb25Nb2RlbChjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> InstructBlipVisionConfig, InstructBlipVisionModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipVisionConfig with Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = InstructBlipVisionConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipVisionModel (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipVisionModel(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`}}),{c(){f=s("p"),C=o("Example:"),_=d(),y(h.$$.fragment)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Example:"),g.forEach(n),_=p(c),I(h.$$.fragment,c)},m(c,g){u(c,f,g),e(f,C),u(c,_,g),B(h,c,g),v=!0},p:or,i(c){v||(w(h.$$.fragment,c),v=!0)},o(c){M(h.$$.fragment,c),v=!1},d(c){c&&n(f),c&&n(_),T(h,c)}}}function Pc(x){let f,C,_,h,v;return h=new rr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEluc3RydWN0QmxpcFFGb3JtZXJDb25maWclMkMlMjBJbnN0cnVjdEJsaXBRRm9ybWVyTW9kZWwlMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwSW5zdHJ1Y3RCTElQJTIwU2FsZXNmb3JjZSUyRmluc3RydWN0LWJsaXAtZmxhbi10NSUyMHN0eWxlJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBJbnN0cnVjdEJsaXBRRm9ybWVyQ29uZmlnKCklMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwbW9kZWwlMjAod2l0aCUyMHJhbmRvbSUyMHdlaWdodHMpJTIwZnJvbSUyMHRoZSUyMFNhbGVzZm9yY2UlMkZpbnN0cnVjdC1ibGlwLWZsYW4tdDUlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFtb2RlbCUyMCUzRCUyMEluc3RydWN0QmxpcFFGb3JtZXJNb2RlbChjb25maWd1cmF0aW9uKSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> InstructBlipQFormerConfig, InstructBlipQFormerModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBLIP Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = InstructBlipQFormerConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipQFormerModel(configuration)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`}}),{c(){f=s("p"),C=o("Examples:"),_=d(),y(h.$$.fragment)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Examples:"),g.forEach(n),_=p(c),I(h.$$.fragment,c)},m(c,g){u(c,f,g),e(f,C),u(c,_,g),B(h,c,g),v=!0},p:or,i(c){v||(w(h.$$.fragment,c),v=!0)},o(c){M(h.$$.fragment,c),v=!1},d(c){c&&n(f),c&&n(_),T(h,c)}}}function xc(x){let f,C,_,h,v;return{c(){f=s("p"),C=o("Although the recipe for forward pass needs to be defined within this function, one should call the "),_=s("code"),h=o("Module"),v=o(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Although the recipe for forward pass needs to be defined within this function, one should call the "),_=a(g,"CODE",{});var G=i(_);h=r(G,"Module"),G.forEach(n),v=r(g,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),g.forEach(n)},m(c,g){u(c,f,g),e(f,C),e(f,_),e(_,h),e(f,v)},d(c){c&&n(f)}}}function Jc(x){let f,C,_,h,v;return{c(){f=s("p"),C=o("Although the recipe for forward pass needs to be defined within this function, one should call the "),_=s("code"),h=o("Module"),v=o(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Although the recipe for forward pass needs to be defined within this function, one should call the "),_=a(g,"CODE",{});var G=i(_);h=r(G,"Module"),G.forEach(n),v=r(g,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),g.forEach(n)},m(c,g){u(c,f,g),e(f,C),e(f,_),e(_,h),e(f,v)},d(c){c&&n(f)}}}function zc(x){let f,C,_,h,v;return h=new rr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEluc3RydWN0QmxpcFByb2Nlc3NvciUyQyUyMEluc3RydWN0QmxpcEZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEElMEFtb2RlbCUyMCUzRCUyMEluc3RydWN0QmxpcEZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbi5mcm9tX3ByZXRyYWluZWQoJTIyU2FsZXNmb3JjZSUyRmluc3RydWN0YmxpcC12aWN1bmEtN2IlMjIpJTBBcHJvY2Vzc29yJTIwJTNEJTIwSW5zdHJ1Y3RCbGlwUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJTYWxlc2ZvcmNlJTJGaW5zdHJ1Y3RibGlwLXZpY3VuYS03YiUyMiklMEElMEFkZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTIyJTIwaWYlMjB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpJTIwZWxzZSUyMCUyMmNwdSUyMiUwQW1vZGVsLnRvKGRldmljZSklMEF1cmwlMjAlM0QlMjAlMjJodHRwcyUzQSUyRiUyRnJhdy5naXRodWJ1c2VyY29udGVudC5jb20lMkZzYWxlc2ZvcmNlJTJGTEFWSVMlMkZtYWluJTJGZG9jcyUyRl9zdGF0aWMlMkZDb25mdXNpbmctUGljdHVyZXMuanBnJTIyJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpLmNvbnZlcnQoJTIyUkdCJTIyKSUwQXByb21wdCUyMCUzRCUyMCUyMldoYXQlMjBpcyUyMHVudXN1YWwlMjBhYm91dCUyMHRoaXMlMjBpbWFnZSUzRiUyMiUwQWlucHV0cyUyMCUzRCUyMHByb2Nlc3NvcihpbWFnZXMlM0RpbWFnZSUyQyUyMHRleHQlM0Rwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhkZXZpY2UpJTBBJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCUwQSUyMCUyMCUyMCUyMCoqaW5wdXRzJTJDJTBBJTIwJTIwJTIwJTIwZG9fc2FtcGxlJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBudW1fYmVhbXMlM0Q1JTJDJTBBJTIwJTIwJTIwJTIwbWF4X2xlbmd0aCUzRDI1NiUyQyUwQSUyMCUyMCUyMCUyMG1pbl9sZW5ndGglM0QxJTJDJTBBJTIwJTIwJTIwJTIwdG9wX3AlM0QwLjklMkMlMEElMjAlMjAlMjAlMjByZXBldGl0aW9uX3BlbmFsdHklM0QxLjUlMkMlMEElMjAlMjAlMjAlMjBsZW5ndGhfcGVuYWx0eSUzRDEuMCUyQyUwQSUyMCUyMCUyMCUyMHRlbXBlcmF0dXJlJTNEMSUyQyUwQSklMEFnZW5lcmF0ZWRfdGV4dCUyMCUzRCUyMHByb2Nlc3Nvci5iYXRjaF9kZWNvZGUob3V0cHV0cyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQuc3RyaXAoKSUwQXByaW50KGdlbmVyYXRlZF90ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> InstructBlipProcessor, InstructBlipForConditionalGeneration
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests

<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;Salesforce/instructblip-vicuna-7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor = InstructBlipProcessor.from_pretrained(<span class="hljs-string">&quot;Salesforce/instructblip-vicuna-7b&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw).convert(<span class="hljs-string">&quot;RGB&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;What is unusual about this image?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor(images=image, text=prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(
<span class="hljs-meta">... </span>    **inputs,
<span class="hljs-meta">... </span>    do_sample=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    num_beams=<span class="hljs-number">5</span>,
<span class="hljs-meta">... </span>    max_length=<span class="hljs-number">256</span>,
<span class="hljs-meta">... </span>    min_length=<span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>    top_p=<span class="hljs-number">0.9</span>,
<span class="hljs-meta">... </span>    repetition_penalty=<span class="hljs-number">1.5</span>,
<span class="hljs-meta">... </span>    length_penalty=<span class="hljs-number">1.0</span>,
<span class="hljs-meta">... </span>    temperature=<span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_text = processor.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>].strip()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(generated_text)
The unusual aspect of this image <span class="hljs-keyword">is</span> that a man <span class="hljs-keyword">is</span> ironing clothes on the back of a yellow SUV, which <span class="hljs-keyword">is</span> parked <span class="hljs-keyword">in</span> the middle of a busy city street. This <span class="hljs-keyword">is</span> an unconventional approach to ironing clothes, <span class="hljs-keyword">as</span> it requires the man to balance himself <span class="hljs-keyword">and</span> his ironing equipment on top of the vehicle <span class="hljs-keyword">while</span> navigating through traffic. Additionally, the presence of taxis <span class="hljs-keyword">and</span> other vehicles <span class="hljs-keyword">in</span> the scene further emphasizes the unusual nature of this situation.`}}),{c(){f=s("p"),C=o("Examples:"),_=d(),y(h.$$.fragment)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Examples:"),g.forEach(n),_=p(c),I(h.$$.fragment,c)},m(c,g){u(c,f,g),e(f,C),u(c,_,g),B(h,c,g),v=!0},p:or,i(c){v||(w(h.$$.fragment,c),v=!0)},o(c){M(h.$$.fragment,c),v=!1},d(c){c&&n(f),c&&n(_),T(h,c)}}}function Uc(x){let f,C,_,h,v,c,g,G,sr,co,Y,fe,mn,Ue,ar,un,ir,po,V,lr,We,cr,dr,Tt,pr,mr,mo,$t,ur,uo,Ct,fn,fr,fo,kt,hr,ho,Et,Ze,gr,jt,_r,vr,go,he,qi,_o,Qe,br,Ft,yr,vo,A,Ir,qe,Br,wr,Ge,Mr,Tr,bo,K,ge,hn,Ne,$r,gn,Cr,yo,F,Se,kr,L,Pt,Er,jr,xt,Fr,Pr,Ve,xr,Jr,zr,ee,Ur,Jt,Wr,Zr,zt,Qr,qr,Gr,_e,Nr,ve,Ae,Sr,Le,Vr,Ut,Ar,Lr,Io,te,be,_n,Re,Rr,vn,Dr,Bo,W,De,Or,ne,Xr,Wt,Hr,Yr,Oe,Kr,es,ts,oe,ns,Zt,os,rs,Qt,ss,as,is,ye,wo,re,Ie,bn,Xe,ls,yn,cs,Mo,Z,He,ds,Q,ps,qt,ms,us,Ye,fs,hs,Gt,gs,_s,Nt,vs,bs,ys,se,Is,St,Bs,ws,Vt,Ms,Ts,$s,Be,To,ae,we,In,Ke,Cs,Bn,ks,$o,P,et,Es,wn,js,Fs,J,At,Ps,xs,Lt,Js,zs,Rt,Us,Ws,Mn,Zs,Qs,Dt,qs,Gs,Ns,Me,tt,Ss,nt,Vs,Ot,As,Ls,Rs,Te,ot,Ds,rt,Os,Xt,Xs,Hs,Co,ie,$e,Tn,st,Ys,$n,Ks,ko,le,at,ea,R,it,ta,ce,na,Ht,oa,ra,Cn,sa,aa,ia,Ce,Eo,de,ke,kn,lt,la,En,ca,jo,N,ct,da,jn,pa,ma,D,dt,ua,k,fa,Fn,ha,ga,Pn,_a,va,xn,ba,ya,Jn,Ia,Ba,zn,wa,Ma,Un,Ta,$a,Wn,Ca,ka,Ea,pt,mt,ja,Zn,Fa,Pa,xa,b,Ja,Qn,za,Ua,qn,Wa,Za,Gn,Qa,qa,Nn,Ga,Na,Sn,Sa,Va,Vn,Aa,La,An,Ra,Da,Ln,Oa,Xa,Rn,Ha,Ya,Dn,Ka,ei,On,ti,ni,Xn,oi,ri,Hn,si,ai,Yn,ii,li,Fo,pe,Ee,Kn,ut,ci,eo,di,Po,E,ft,pi,to,mi,ui,ht,fi,no,hi,gi,_i,gt,vi,Yt,bi,yi,Ii,_t,Bi,vt,wi,Mi,Ti,q,bt,$i,me,Ci,Kt,ki,Ei,oo,ji,Fi,Pi,je,xi,Fe,Ji,Pe,yt,zi,It,Ui,ro,Wi,Zi,xo;return c=new ue({}),Ue=new ue({}),Ne=new ue({}),Se=new U({props:{name:"class transformers.InstructBlipConfig",anchor:"transformers.InstructBlipConfig",parameters:[{name:"vision_config",val:" = None"},{name:"qformer_config",val:" = None"},{name:"text_config",val:" = None"},{name:"num_query_tokens",val:" = 32"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipConfig.vision_config",description:`<strong>vision_config</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize <a href="/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipVisionConfig">InstructBlipVisionConfig</a>.`,name:"vision_config"},{anchor:"transformers.InstructBlipConfig.qformer_config",description:`<strong>qformer_config</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize <a href="/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipQFormerConfig">InstructBlipQFormerConfig</a>.`,name:"qformer_config"},{anchor:"transformers.InstructBlipConfig.text_config",description:`<strong>text_config</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize any <a href="/docs/transformers/v4.33.0/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a>.`,name:"text_config"},{anchor:"transformers.InstructBlipConfig.num_query_tokens",description:`<strong>num_query_tokens</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
The number of query tokens passed through the Transformer.`,name:"num_query_tokens"},{anchor:"transformers.InstructBlipConfig.kwargs",description:`<strong>kwargs</strong> (<em>optional</em>) &#x2014;
Dictionary of keyword arguments.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/configuration_instructblip.py#L252"}}),_e=new nr({props:{anchor:"transformers.InstructBlipConfig.example",$$slots:{default:[jc]},$$scope:{ctx:x}}}),Ae=new U({props:{name:"from_vision_qformer_text_configs",anchor:"transformers.InstructBlipConfig.from_vision_qformer_text_configs",parameters:[{name:"vision_config",val:": InstructBlipVisionConfig"},{name:"qformer_config",val:": InstructBlipQFormerConfig"},{name:"text_config",val:": PretrainedConfig"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/configuration_instructblip.py#L337",returnDescription:`
<p>An instance of a configuration object</p>
`,returnType:`
<p><a
  href="/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipConfig"
>InstructBlipConfig</a></p>
`}}),Re=new ue({}),De=new U({props:{name:"class transformers.InstructBlipVisionConfig",anchor:"transformers.InstructBlipVisionConfig",parameters:[{name:"hidden_size",val:" = 1408"},{name:"intermediate_size",val:" = 6144"},{name:"num_hidden_layers",val:" = 39"},{name:"num_attention_heads",val:" = 16"},{name:"image_size",val:" = 224"},{name:"patch_size",val:" = 14"},{name:"hidden_act",val:" = 'gelu'"},{name:"layer_norm_eps",val:" = 1e-06"},{name:"attention_dropout",val:" = 0.0"},{name:"initializer_range",val:" = 1e-10"},{name:"qkv_bias",val:" = True"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipVisionConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 1408) &#x2014;
Dimensionality of the encoder layers and the pooler layer.`,name:"hidden_size"},{anchor:"transformers.InstructBlipVisionConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 6144) &#x2014;
Dimensionality of the &#x201C;intermediate&#x201D; (i.e., feed-forward) layer in the Transformer encoder.`,name:"intermediate_size"},{anchor:"transformers.InstructBlipVisionConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 39) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.InstructBlipVisionConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 16) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.InstructBlipVisionConfig.image_size",description:`<strong>image_size</strong> (<code>int</code>, <em>optional</em>, defaults to 224) &#x2014;
The size (resolution) of each image.`,name:"image_size"},{anchor:"transformers.InstructBlipVisionConfig.patch_size",description:`<strong>patch_size</strong> (<code>int</code>, <em>optional</em>, defaults to 14) &#x2014;
The size (resolution) of each patch.`,name:"patch_size"},{anchor:"transformers.InstructBlipVisionConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>function</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the encoder and pooler. If string, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;selu&quot;</code> and <code>&quot;gelu_new&quot;</code> \`<code>&quot;gelu&quot;</code> are supported. to 1e-5): The epsilon used by the layer
normalization layers.`,name:"hidden_act"},{anchor:"transformers.InstructBlipVisionConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-6) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.InstructBlipVisionConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_dropout"},{anchor:"transformers.InstructBlipVisionConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-10) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.InstructBlipVisionConfig.qkv_bias",description:`<strong>qkv_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to add a bias to the queries and values in the self-attention layers.`,name:"qkv_bias"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/configuration_instructblip.py#L33"}}),ye=new nr({props:{anchor:"transformers.InstructBlipVisionConfig.example",$$slots:{default:[Fc]},$$scope:{ctx:x}}}),Xe=new ue({}),He=new U({props:{name:"class transformers.InstructBlipQFormerConfig",anchor:"transformers.InstructBlipQFormerConfig",parameters:[{name:"vocab_size",val:" = 30522"},{name:"hidden_size",val:" = 768"},{name:"num_hidden_layers",val:" = 12"},{name:"num_attention_heads",val:" = 12"},{name:"intermediate_size",val:" = 3072"},{name:"hidden_act",val:" = 'gelu'"},{name:"hidden_dropout_prob",val:" = 0.1"},{name:"attention_probs_dropout_prob",val:" = 0.1"},{name:"max_position_embeddings",val:" = 512"},{name:"initializer_range",val:" = 0.02"},{name:"layer_norm_eps",val:" = 1e-12"},{name:"pad_token_id",val:" = 0"},{name:"position_embedding_type",val:" = 'absolute'"},{name:"cross_attention_frequency",val:" = 2"},{name:"encoder_hidden_size",val:" = 1408"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipQFormerConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 30522) &#x2014;
Vocabulary size of the Q-Former model. Defines the number of different tokens that can be represented by
the <code>inputs_ids</code> passed when calling the model.`,name:"vocab_size"},{anchor:"transformers.InstructBlipQFormerConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 768) &#x2014;
Dimensionality of the encoder layers and the pooler layer.`,name:"hidden_size"},{anchor:"transformers.InstructBlipQFormerConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 12) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.InstructBlipQFormerConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 12) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.InstructBlipQFormerConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 3072) &#x2014;
Dimensionality of the &#x201C;intermediate&#x201D; (often named feed-forward) layer in the Transformer encoder.`,name:"intermediate_size"},{anchor:"transformers.InstructBlipQFormerConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>Callable</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the encoder and pooler. If string, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;silu&quot;</code> and <code>&quot;gelu_new&quot;</code> are supported.`,name:"hidden_act"},{anchor:"transformers.InstructBlipQFormerConfig.hidden_dropout_prob",description:`<strong>hidden_dropout_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.`,name:"hidden_dropout_prob"},{anchor:"transformers.InstructBlipQFormerConfig.attention_probs_dropout_prob",description:`<strong>attention_probs_dropout_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_probs_dropout_prob"},{anchor:"transformers.InstructBlipQFormerConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to 512) &#x2014;
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).`,name:"max_position_embeddings"},{anchor:"transformers.InstructBlipQFormerConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 0.02) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.InstructBlipQFormerConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-12) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.InstructBlipQFormerConfig.position_embedding_type",description:`<strong>position_embedding_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;absolute&quot;</code>) &#x2014;
Type of position embedding. Choose one of <code>&quot;absolute&quot;</code>, <code>&quot;relative_key&quot;</code>, <code>&quot;relative_key_query&quot;</code>. For
positional embeddings use <code>&quot;absolute&quot;</code>. For more information on <code>&quot;relative_key&quot;</code>, please refer to
<a href="https://arxiv.org/abs/1803.02155" rel="nofollow">Self-Attention with Relative Position Representations (Shaw et al.)</a>.
For more information on <code>&quot;relative_key_query&quot;</code>, please refer to <em>Method 4</em> in <a href="https://arxiv.org/abs/2009.13658" rel="nofollow">Improve Transformer Models
with Better Relative Position Embeddings (Huang et al.)</a>.`,name:"position_embedding_type"},{anchor:"transformers.InstructBlipQFormerConfig.cross_attention_frequency",description:`<strong>cross_attention_frequency</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
The frequency of adding cross-attention to the Transformer layers.`,name:"cross_attention_frequency"},{anchor:"transformers.InstructBlipQFormerConfig.encoder_hidden_size",description:`<strong>encoder_hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 1408) &#x2014;
The hidden size of the hidden states for cross-attention.`,name:"encoder_hidden_size"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/configuration_instructblip.py#L134"}}),Be=new nr({props:{anchor:"transformers.InstructBlipQFormerConfig.example",$$slots:{default:[Pc]},$$scope:{ctx:x}}}),Ke=new ue({}),et=new U({props:{name:"class transformers.InstructBlipProcessor",anchor:"transformers.InstructBlipProcessor",parameters:[{name:"image_processor",val:""},{name:"tokenizer",val:""},{name:"qformer_tokenizer",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipProcessor.image_processor",description:`<strong>image_processor</strong> (<code>BlipImageProcessor</code>) &#x2014;
An instance of <a href="/docs/transformers/v4.33.0/en/model_doc/blip#transformers.BlipImageProcessor">BlipImageProcessor</a>. The image processor is a required input.`,name:"image_processor"},{anchor:"transformers.InstructBlipProcessor.tokenizer",description:"<strong>tokenizer</strong> (<code>AutoTokenizer</code>) &#x2014;\nAn instance of [&#x2018;PreTrainedTokenizer`]. The tokenizer is a required input.",name:"tokenizer"},{anchor:"transformers.InstructBlipProcessor.qformer_tokenizer",description:"<strong>qformer_tokenizer</strong> (<code>AutoTokenizer</code>) &#x2014;\nAn instance of [&#x2018;PreTrainedTokenizer`]. The Q-Former tokenizer is a required input.",name:"qformer_tokenizer"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/processing_instructblip.py#L30"}}),tt=new U({props:{name:"batch_decode",anchor:"transformers.InstructBlipProcessor.batch_decode",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/processing_instructblip.py#L135"}}),ot=new U({props:{name:"decode",anchor:"transformers.InstructBlipProcessor.decode",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/processing_instructblip.py#L143"}}),st=new ue({}),at=new U({props:{name:"class transformers.InstructBlipVisionModel",anchor:"transformers.InstructBlipVisionModel",parameters:[{name:"config",val:": InstructBlipVisionConfig"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/modeling_instructblip.py#L500"}}),it=new U({props:{name:"forward",anchor:"transformers.InstructBlipVisionModel.forward",parameters:[{name:"pixel_values",val:": typing.Optional[torch.FloatTensor] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.InstructBlipVisionModel.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using <a href="/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See
<code>InstructBlipProcessor.__call__()</code> for details.`,name:"pixel_values"},{anchor:"transformers.InstructBlipVisionModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.InstructBlipVisionModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.InstructBlipVisionModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.33.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/modeling_instructblip.py#L515",returnDescription:`
<p>A <a
  href="/docs/transformers/v4.33.0/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPooling"
>transformers.modeling_outputs.BaseModelOutputWithPooling</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<code>&lt;class 'transformers.models.instructblip.configuration_instructblip.InstructBlipVisionConfig'&gt;</code>) and inputs.</p>
<ul>
<li>
<p><strong>last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) \u2014 Sequence of hidden-states at the output of the last layer of the model.</p>
</li>
<li>
<p><strong>pooler_output</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, hidden_size)</code>) \u2014 Last layer hidden-state of the first token of the sequence (classification token) after further processing
through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns
the classification token after processing through a linear layer and a tanh activation function. The linear
layer weights are trained from the next sentence prediction (classification) objective during pretraining.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/v4.33.0/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPooling"
>transformers.modeling_outputs.BaseModelOutputWithPooling</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),Ce=new wc({props:{$$slots:{default:[xc]},$$scope:{ctx:x}}}),lt=new ue({}),ct=new U({props:{name:"class transformers.InstructBlipQFormerModel",anchor:"transformers.InstructBlipQFormerModel",parameters:[{name:"config",val:": InstructBlipQFormerConfig"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/modeling_instructblip.py#L1052"}}),dt=new U({props:{name:"forward",anchor:"transformers.InstructBlipQFormerModel.forward",parameters:[{name:"input_ids",val:": LongTensor"},{name:"attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"position_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"query_embeds",val:": typing.Optional[torch.Tensor] = None"},{name:"head_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"encoder_hidden_states",val:": typing.Optional[torch.FloatTensor] = None"},{name:"encoder_attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"past_key_values",val:": typing.Optional[typing.Tuple[typing.Tuple[torch.FloatTensor]]] = None"},{name:"use_cache",val:": typing.Optional[bool] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/modeling_instructblip.py#L1125"}}),ut=new ue({}),ft=new U({props:{name:"class transformers.InstructBlipForConditionalGeneration",anchor:"transformers.InstructBlipForConditionalGeneration",parameters:[{name:"config",val:": InstructBlipConfig"}],parametersDescription:[{anchor:"transformers.InstructBlipForConditionalGeneration.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipConfig">InstructBlipConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/v4.33.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/modeling_instructblip.py#L1258"}}),bt=new U({props:{name:"forward",anchor:"transformers.InstructBlipForConditionalGeneration.forward",parameters:[{name:"pixel_values",val:": FloatTensor"},{name:"qformer_input_ids",val:": FloatTensor"},{name:"qformer_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"input_ids",val:": typing.Optional[torch.FloatTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"decoder_input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"decoder_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.InstructBlipForConditionalGeneration.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using <a href="/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See
<code>InstructBlipProcessor.__call__()</code> for details.`,name:"pixel_values"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.qformer_input_ids",description:`<strong>qformer_input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
to serve as text prompt, which the Q-Former model will encode.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See <code>InstructBlipProcessor.__call__()</code> for
details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"qformer_input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.qformer_attention_mask",description:`<strong>qformer_attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"qformer_attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be
provided to serve as text prompt, which the language model can continue.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See <code>InstructBlipProcessor.__call__()</code> for
details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.decoder_input_ids",description:`<strong>decoder_input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of decoder input sequence tokens in the vocabulary of the language model. Only relevant in case an
encoder-decoder language model (like T5) is used.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.33.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/v4.33.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.33.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details. <a href="../glossary#decoder-input-ids">What are decoder input IDs?</a>`,name:"decoder_input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.decoder_attention_mask",description:`<strong>decoder_attention_mask</strong> (<code>torch.BoolTensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Default behavior: generate a tensor that ignores pad tokens in <code>decoder_input_ids</code>. Causal mask will also
be used by default.</p>
<p>Only relevant in case an encoder-decoder language model (like T5) is used.`,name:"decoder_attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.33.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) &#x2014;
Labels for computing the language modeling loss. Indices should be in <code>[-100, 0, ..., config.vocab_size - 1]</code>. All labels set to <code>-100</code> are ignored (masked), the loss is only computed for labels in <code>[0, ..., config.vocab_size]</code>`,name:"labels"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/modeling_instructblip.py#L1331",returnDescription:`
<p>A <code>transformers.models.instructblip.modeling_instructblip.InstructBlipForConditionalGenerationModelOutput</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<code>&lt;class 'transformers.models.instructblip.configuration_instructblip.InstructBlipVisionConfig'&gt;</code>) and inputs.</p>
<ul>
<li><strong>loss</strong> (<code>torch.FloatTensor</code>, <em>optional</em>, returned when <code>labels</code> is provided, <code>torch.FloatTensor</code> of shape <code>(1,)</code>) \u2014 Language modeling loss from the language model.</li>
<li><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) \u2014 Prediction scores of the language modeling head of the language model.</li>
<li><strong>vision_outputs</strong> (<code>BaseModelOutputWithPooling</code>) \u2014 Outputs of the vision encoder.</li>
<li><strong>qformer_outputs</strong> (<code>BaseModelOutputWithPoolingAndCrossAttentions</code>) \u2014 Outputs of the Q-Former (Querying Transformer).</li>
<li><strong>language_model_outputs</strong> (<code>CausalLMOutputWithPast</code> or <code>Seq2SeqLMOutput</code>) \u2014 Outputs of the language model.</li>
</ul>
`,returnType:`
<p><code>transformers.models.instructblip.modeling_instructblip.InstructBlipForConditionalGenerationModelOutput</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),je=new wc({props:{$$slots:{default:[Jc]},$$scope:{ctx:x}}}),Fe=new nr({props:{anchor:"transformers.InstructBlipForConditionalGeneration.forward.example",$$slots:{default:[zc]},$$scope:{ctx:x}}}),yt=new U({props:{name:"generate",anchor:"transformers.InstructBlipForConditionalGeneration.generate",parameters:[{name:"pixel_values",val:": FloatTensor"},{name:"qformer_input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"qformer_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"**generate_kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipForConditionalGeneration.generate.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape (batch_size, num_channels, height, width)) &#x2014;
Input images to be processed.`,name:"pixel_values"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.qformer_input_ids",description:`<strong>qformer_input_ids</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
The sequence used as a prompt to be fed to the Q-Former module.`,name:"qformer_input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.qformer_attention_mask",description:`<strong>qformer_attention_mask</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices.`,name:"qformer_attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
The sequence used as a prompt for the generation.`,name:"input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices.`,name:"attention_mask"}],source:"https://github.com/huggingface/transformers/blob/v4.33.0/src/transformers/models/instructblip/modeling_instructblip.py#L1486",returnDescription:`
<p>A list of strings of length batch_size * num_captions.</p>
`,returnType:`
<p>captions (list)</p>
`}}),{c(){f=s("meta"),C=d(),_=s("h1"),h=s("a"),v=s("span"),y(c.$$.fragment),g=d(),G=s("span"),sr=o("InstructBLIP"),co=d(),Y=s("h2"),fe=s("a"),mn=s("span"),y(Ue.$$.fragment),ar=d(),un=s("span"),ir=o("Overview"),po=d(),V=s("p"),lr=o("The InstructBLIP model was proposed in "),We=s("a"),cr=o("InstructBLIP: Towards General-purpose Vision-Language Models with Instruction Tuning"),dr=o(` by Wenliang Dai, Junnan Li, Dongxu Li, Anthony Meng Huat Tiong, Junqi Zhao, Weisheng Wang, Boyang Li, Pascale Fung, Steven Hoi.
InstructBLIP leverages the `),Tt=s("a"),pr=o("BLIP-2"),mr=o(" architecture for visual instruction tuning."),mo=d(),$t=s("p"),ur=o("The abstract from the paper is the following:"),uo=d(),Ct=s("p"),fn=s("em"),fr=o("General-purpose language models that can solve various language-domain tasks have emerged driven by the pre-training and instruction-tuning pipeline. However, building general-purpose vision-language models is challenging due to the increased task discrepancy introduced by the additional visual input. Although vision-language pre-training has been widely studied, vision-language instruction tuning remains relatively less explored. In this paper, we conduct a systematic and comprehensive study on vision-language instruction tuning based on the pre-trained BLIP-2 models. We gather a wide variety of 26 publicly available datasets, transform them into instruction tuning format and categorize them into two clusters for held-in instruction tuning and held-out zero-shot evaluation. Additionally, we introduce instruction-aware visual feature extraction, a crucial method that enables the model to extract informative features tailored to the given instruction. The resulting InstructBLIP models achieve state-of-the-art zero-shot performance across all 13 held-out datasets, substantially outperforming BLIP-2 and the larger Flamingo. Our models also lead to state-of-the-art performance when finetuned on individual downstream tasks (e.g., 90.7% accuracy on ScienceQA IMG). Furthermore, we qualitatively demonstrate the advantages of InstructBLIP over concurrent multimodal models."),fo=d(),kt=s("p"),hr=o("Tips:"),ho=d(),Et=s("ul"),Ze=s("li"),gr=o("InstructBLIP uses the same architecture as "),jt=s("a"),_r=o("BLIP-2"),vr=o(" with a tiny but important difference: it also feeds the text prompt (instruction) to the Q-Former."),go=d(),he=s("img"),_o=d(),Qe=s("small"),br=o("InstructBLIP architecture. Taken from the "),Ft=s("a"),yr=o("original paper."),vo=d(),A=s("p"),Ir=o("This model was contributed by "),qe=s("a"),Br=o("nielsr"),wr=o(`.
The original code can be found `),Ge=s("a"),Mr=o("here"),Tr=o("."),bo=d(),K=s("h2"),ge=s("a"),hn=s("span"),y(Ne.$$.fragment),$r=d(),gn=s("span"),Cr=o("InstructBlipConfig"),yo=d(),F=s("div"),y(Se.$$.fragment),kr=d(),L=s("p"),Pt=s("a"),Er=o("InstructBlipConfig"),jr=o(` is the configuration class to store the configuration of a
`),xt=s("a"),Fr=o("InstructBlipForConditionalGeneration"),Pr=o(`. It is used to instantiate a InstructBLIP model according to the specified
arguments, defining the vision model, Q-Former model and language model configs. Instantiating a configuration with
the defaults will yield a similar configuration to that of the InstructBLIP
`),Ve=s("a"),xr=o("Salesforce/instruct-blip-flan-t5"),Jr=o(" architecture."),zr=d(),ee=s("p"),Ur=o("Configuration objects inherit from "),Jt=s("a"),Wr=o("PretrainedConfig"),Zr=o(` and can be used to control the model outputs. Read the
documentation from `),zt=s("a"),Qr=o("PretrainedConfig"),qr=o(" for more information."),Gr=d(),y(_e.$$.fragment),Nr=d(),ve=s("div"),y(Ae.$$.fragment),Sr=d(),Le=s("p"),Vr=o("Instantiate a "),Ut=s("a"),Ar=o("InstructBlipConfig"),Lr=o(` (or a derived class) from a InstructBLIP vision model, Q-Former and
language model configurations.`),Io=d(),te=s("h2"),be=s("a"),_n=s("span"),y(Re.$$.fragment),Rr=d(),vn=s("span"),Dr=o("InstructBlipVisionConfig"),Bo=d(),W=s("div"),y(De.$$.fragment),Or=d(),ne=s("p"),Xr=o("This is the configuration class to store the configuration of a "),Wt=s("a"),Hr=o("InstructBlipVisionModel"),Yr=o(`. It is used to
instantiate a InstructBLIP vision encoder according to the specified arguments, defining the model architecture.
Instantiating a configuration defaults will yield a similar configuration to that of the InstructBLIP
`),Oe=s("a"),Kr=o("Salesforce/instruct-blip-flan-t5"),es=o(" architecture."),ts=d(),oe=s("p"),ns=o("Configuration objects inherit from "),Zt=s("a"),os=o("PretrainedConfig"),rs=o(` and can be used to control the model outputs. Read the
documentation from `),Qt=s("a"),ss=o("PretrainedConfig"),as=o(" for more information."),is=d(),y(ye.$$.fragment),wo=d(),re=s("h2"),Ie=s("a"),bn=s("span"),y(Xe.$$.fragment),ls=d(),yn=s("span"),cs=o("InstructBlipQFormerConfig"),Mo=d(),Z=s("div"),y(He.$$.fragment),ds=d(),Q=s("p"),ps=o("This is the configuration class to store the configuration of a "),qt=s("a"),ms=o("InstructBlipQFormerModel"),us=o(`. It is used to
instantiate a InstructBLIP Querying Transformer (Q-Former) model according to the specified arguments, defining the
model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
the InstructBLIP `),Ye=s("a"),fs=o("Salesforce/instruct-blip-flan-t5"),hs=o(`
architecture. Configuration objects inherit from `),Gt=s("a"),gs=o("PretrainedConfig"),_s=o(` and can be used to control the model outputs.
Read the documentation from `),Nt=s("a"),vs=o("PretrainedConfig"),bs=o(" for more information."),ys=d(),se=s("p"),Is=o("Note that "),St=s("a"),Bs=o("InstructBlipQFormerModel"),ws=o(" is very similar to "),Vt=s("a"),Ms=o("BertLMHeadModel"),Ts=o(" with interleaved cross-attention."),$s=d(),y(Be.$$.fragment),To=d(),ae=s("h2"),we=s("a"),In=s("span"),y(Ke.$$.fragment),Cs=d(),Bn=s("span"),ks=o("InstructBlipProcessor"),$o=d(),P=s("div"),y(et.$$.fragment),Es=d(),wn=s("p"),js=o(`Constructs an InstructBLIP processor which wraps a BLIP image processor and a LLaMa/T5 tokenizer into a single
processor.`),Fs=d(),J=s("p"),At=s("a"),Ps=o("InstructBlipProcessor"),xs=o(" offers all the functionalities of "),Lt=s("a"),Js=o("BlipImageProcessor"),zs=o(" and "),Rt=s("a"),Us=o("AutoTokenizer"),Ws=o(`. See the
docstring of `),Mn=s("code"),Zs=o("__call__()"),Qs=o(" and "),Dt=s("a"),qs=o("decode()"),Gs=o(" for more information."),Ns=d(),Me=s("div"),y(tt.$$.fragment),Ss=d(),nt=s("p"),Vs=o("This method forwards all its arguments to PreTrainedTokenizer\u2019s "),Ot=s("a"),As=o("batch_decode()"),Ls=o(`. Please
refer to the docstring of this method for more information.`),Rs=d(),Te=s("div"),y(ot.$$.fragment),Ds=d(),rt=s("p"),Os=o("This method forwards all its arguments to PreTrainedTokenizer\u2019s "),Xt=s("a"),Xs=o("decode()"),Hs=o(`. Please refer
to the docstring of this method for more information.`),Co=d(),ie=s("h2"),$e=s("a"),Tn=s("span"),y(st.$$.fragment),Ys=d(),$n=s("span"),Ks=o("InstructBlipVisionModel"),ko=d(),le=s("div"),y(at.$$.fragment),ea=d(),R=s("div"),y(it.$$.fragment),ta=d(),ce=s("p"),na=o("The "),Ht=s("a"),oa=o("InstructBlipVisionModel"),ra=o(" forward method, overrides the "),Cn=s("code"),sa=o("__call__"),aa=o(" special method."),ia=d(),y(Ce.$$.fragment),Eo=d(),de=s("h2"),ke=s("a"),kn=s("span"),y(lt.$$.fragment),la=d(),En=s("span"),ca=o("InstructBlipQFormerModel"),jo=d(),N=s("div"),y(ct.$$.fragment),da=d(),jn=s("p"),pa=o(`Querying Transformer (Q-Former), used in InstructBLIP. Slightly modified from BLIP-2 as it also takes the
instruction as input.`),ma=d(),D=s("div"),y(dt.$$.fragment),ua=d(),k=s("p"),fa=o("encoder_hidden_states  ("),Fn=s("code"),ha=o("torch.FloatTensor"),ga=o(" of shape "),Pn=s("code"),_a=o("(batch_size, sequence_length, hidden_size)"),va=o(", "),xn=s("em"),ba=o("optional"),ya=o(`):
Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
the model is configured as a decoder.
encoder_attention_mask (`),Jn=s("code"),Ia=o("torch.FloatTensor"),Ba=o(" of shape "),zn=s("code"),wa=o("(batch_size, sequence_length)"),Ma=o(", "),Un=s("em"),Ta=o("optional"),$a=o(`):
Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
the cross-attention if the model is configured as a decoder. Mask values selected in `),Wn=s("code"),Ca=o("[0, 1]"),ka=o(":"),Ea=d(),pt=s("ul"),mt=s("li"),ja=o("1 for tokens that are "),Zn=s("strong"),Fa=o("not masked"),Pa=o(","),xa=d(),b=s("li"),Ja=o("0 for tokens that are "),Qn=s("strong"),za=o("masked"),Ua=o(`.
past_key_values (`),qn=s("code"),Wa=o("tuple(tuple(torch.FloatTensor))"),Za=o(" of length "),Gn=s("code"),Qa=o("config.n_layers"),qa=o(` with each tuple having 4 tensors of:
shape `),Nn=s("code"),Ga=o("(batch_size, num_heads, sequence_length - 1, embed_size_per_head)"),Na=o(`): Contains precomputed key and
value hidden states of the attention blocks. Can be used to speed up decoding. If `),Sn=s("code"),Sa=o("past_key_values"),Va=o(` are
used, the user can optionally input only the last `),Vn=s("code"),Aa=o("decoder_input_ids"),La=o(` (those that don\u2019t have their past key
value states given to this model) of shape `),An=s("code"),Ra=o("(batch_size, 1)"),Da=o(" instead of all "),Ln=s("code"),Oa=o("decoder_input_ids"),Xa=o(` of shape
`),Rn=s("code"),Ha=o("(batch_size, sequence_length)"),Ya=o(`.
use_cache (`),Dn=s("code"),Ka=o("bool"),ei=o(", "),On=s("em"),ti=o("optional"),ni=o(`):
If set to `),Xn=s("code"),oi=o("True"),ri=o(", "),Hn=s("code"),si=o("past_key_values"),ai=o(` key value states are returned and can be used to speed up decoding (see
`),Yn=s("code"),ii=o("past_key_values"),li=o(")."),Fo=d(),pe=s("h2"),Ee=s("a"),Kn=s("span"),y(ut.$$.fragment),ci=d(),eo=s("span"),di=o("InstructBlipForConditionalGeneration"),Po=d(),E=s("div"),y(ft.$$.fragment),pi=d(),to=s("p"),mi=o(`InstructBLIP Model for generating text given an image and an optional text prompt. The model consists of a vision
encoder, Querying Transformer (Q-Former) and a language model.`),ui=d(),ht=s("p"),fi=o("One can optionally pass "),no=s("code"),hi=o("input_ids"),gi=o(` to the model, which serve as a text prompt, to make the language model continue
the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.`),_i=d(),gt=s("p"),vi=o("This model inherits from "),Yt=s("a"),bi=o("PreTrainedModel"),yi=o(`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),Ii=d(),_t=s("p"),Bi=o("This model is also a PyTorch "),vt=s("a"),wi=o("torch.nn.Module"),Mi=o(` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),Ti=d(),q=s("div"),y(bt.$$.fragment),$i=d(),me=s("p"),Ci=o("The "),Kt=s("a"),ki=o("InstructBlipForConditionalGeneration"),Ei=o(" forward method, overrides the "),oo=s("code"),ji=o("__call__"),Fi=o(" special method."),Pi=d(),y(je.$$.fragment),xi=d(),y(Fe.$$.fragment),Ji=d(),Pe=s("div"),y(yt.$$.fragment),zi=d(),It=s("p"),Ui=o("Overrides "),ro=s("code"),Wi=o("generate"),Zi=o(" function to be able to use the model as a conditional generator."),this.h()},l(t){const m=Cc('[data-svelte="svelte-1phssyn"]',document.head);f=a(m,"META",{name:!0,content:!0}),m.forEach(n),C=p(t),_=a(t,"H1",{class:!0});var Bt=i(_);h=a(Bt,"A",{id:!0,class:!0,href:!0});var so=i(h);v=a(so,"SPAN",{});var ao=i(v);I(c.$$.fragment,ao),ao.forEach(n),so.forEach(n),g=p(Bt),G=a(Bt,"SPAN",{});var io=i(G);sr=r(io,"InstructBLIP"),io.forEach(n),Bt.forEach(n),co=p(t),Y=a(t,"H2",{class:!0});var wt=i(Y);fe=a(wt,"A",{id:!0,class:!0,href:!0});var lo=i(fe);mn=a(lo,"SPAN",{});var Gi=i(mn);I(Ue.$$.fragment,Gi),Gi.forEach(n),lo.forEach(n),ar=p(wt),un=a(wt,"SPAN",{});var Ni=i(un);ir=r(Ni,"Overview"),Ni.forEach(n),wt.forEach(n),po=p(t),V=a(t,"P",{});var en=i(V);lr=r(en,"The InstructBLIP model was proposed in "),We=a(en,"A",{href:!0,rel:!0});var Si=i(We);cr=r(Si,"InstructBLIP: Towards General-purpose Vision-Language Models with Instruction Tuning"),Si.forEach(n),dr=r(en,` by Wenliang Dai, Junnan Li, Dongxu Li, Anthony Meng Huat Tiong, Junqi Zhao, Weisheng Wang, Boyang Li, Pascale Fung, Steven Hoi.
InstructBLIP leverages the `),Tt=a(en,"A",{href:!0});var Vi=i(Tt);pr=r(Vi,"BLIP-2"),Vi.forEach(n),mr=r(en," architecture for visual instruction tuning."),en.forEach(n),mo=p(t),$t=a(t,"P",{});var Ai=i($t);ur=r(Ai,"The abstract from the paper is the following:"),Ai.forEach(n),uo=p(t),Ct=a(t,"P",{});var Li=i(Ct);fn=a(Li,"EM",{});var Ri=i(fn);fr=r(Ri,"General-purpose language models that can solve various language-domain tasks have emerged driven by the pre-training and instruction-tuning pipeline. However, building general-purpose vision-language models is challenging due to the increased task discrepancy introduced by the additional visual input. Although vision-language pre-training has been widely studied, vision-language instruction tuning remains relatively less explored. In this paper, we conduct a systematic and comprehensive study on vision-language instruction tuning based on the pre-trained BLIP-2 models. We gather a wide variety of 26 publicly available datasets, transform them into instruction tuning format and categorize them into two clusters for held-in instruction tuning and held-out zero-shot evaluation. Additionally, we introduce instruction-aware visual feature extraction, a crucial method that enables the model to extract informative features tailored to the given instruction. The resulting InstructBLIP models achieve state-of-the-art zero-shot performance across all 13 held-out datasets, substantially outperforming BLIP-2 and the larger Flamingo. Our models also lead to state-of-the-art performance when finetuned on individual downstream tasks (e.g., 90.7% accuracy on ScienceQA IMG). Furthermore, we qualitatively demonstrate the advantages of InstructBLIP over concurrent multimodal models."),Ri.forEach(n),Li.forEach(n),fo=p(t),kt=a(t,"P",{});var Di=i(kt);hr=r(Di,"Tips:"),Di.forEach(n),ho=p(t),Et=a(t,"UL",{});var Oi=i(Et);Ze=a(Oi,"LI",{});var Jo=i(Ze);gr=r(Jo,"InstructBLIP uses the same architecture as "),jt=a(Jo,"A",{href:!0});var Xi=i(jt);_r=r(Xi,"BLIP-2"),Xi.forEach(n),vr=r(Jo," with a tiny but important difference: it also feeds the text prompt (instruction) to the Q-Former."),Jo.forEach(n),Oi.forEach(n),go=p(t),he=a(t,"IMG",{src:!0,alt:!0,width:!0}),_o=p(t),Qe=a(t,"SMALL",{});var Qi=i(Qe);br=r(Qi,"InstructBLIP architecture. Taken from the "),Ft=a(Qi,"A",{href:!0});var Hi=i(Ft);yr=r(Hi,"original paper."),Hi.forEach(n),Qi.forEach(n),vo=p(t),A=a(t,"P",{});var tn=i(A);Ir=r(tn,"This model was contributed by "),qe=a(tn,"A",{href:!0,rel:!0});var Yi=i(qe);Br=r(Yi,"nielsr"),Yi.forEach(n),wr=r(tn,`.
The original code can be found `),Ge=a(tn,"A",{href:!0,rel:!0});var Ki=i(Ge);Mr=r(Ki,"here"),Ki.forEach(n),Tr=r(tn,"."),tn.forEach(n),bo=p(t),K=a(t,"H2",{class:!0});var zo=i(K);ge=a(zo,"A",{id:!0,class:!0,href:!0});var el=i(ge);hn=a(el,"SPAN",{});var tl=i(hn);I(Ne.$$.fragment,tl),tl.forEach(n),el.forEach(n),$r=p(zo),gn=a(zo,"SPAN",{});var nl=i(gn);Cr=r(nl,"InstructBlipConfig"),nl.forEach(n),zo.forEach(n),yo=p(t),F=a(t,"DIV",{class:!0});var O=i(F);I(Se.$$.fragment,O),kr=p(O),L=a(O,"P",{});var Mt=i(L);Pt=a(Mt,"A",{href:!0});var ol=i(Pt);Er=r(ol,"InstructBlipConfig"),ol.forEach(n),jr=r(Mt,` is the configuration class to store the configuration of a
`),xt=a(Mt,"A",{href:!0});var rl=i(xt);Fr=r(rl,"InstructBlipForConditionalGeneration"),rl.forEach(n),Pr=r(Mt,`. It is used to instantiate a InstructBLIP model according to the specified
arguments, defining the vision model, Q-Former model and language model configs. Instantiating a configuration with
the defaults will yield a similar configuration to that of the InstructBLIP
`),Ve=a(Mt,"A",{href:!0,rel:!0});var sl=i(Ve);xr=r(sl,"Salesforce/instruct-blip-flan-t5"),sl.forEach(n),Jr=r(Mt," architecture."),Mt.forEach(n),zr=p(O),ee=a(O,"P",{});var nn=i(ee);Ur=r(nn,"Configuration objects inherit from "),Jt=a(nn,"A",{href:!0});var al=i(Jt);Wr=r(al,"PretrainedConfig"),al.forEach(n),Zr=r(nn,` and can be used to control the model outputs. Read the
documentation from `),zt=a(nn,"A",{href:!0});var il=i(zt);Qr=r(il,"PretrainedConfig"),il.forEach(n),qr=r(nn," for more information."),nn.forEach(n),Gr=p(O),I(_e.$$.fragment,O),Nr=p(O),ve=a(O,"DIV",{class:!0});var Uo=i(ve);I(Ae.$$.fragment,Uo),Sr=p(Uo),Le=a(Uo,"P",{});var Wo=i(Le);Vr=r(Wo,"Instantiate a "),Ut=a(Wo,"A",{href:!0});var ll=i(Ut);Ar=r(ll,"InstructBlipConfig"),ll.forEach(n),Lr=r(Wo,` (or a derived class) from a InstructBLIP vision model, Q-Former and
language model configurations.`),Wo.forEach(n),Uo.forEach(n),O.forEach(n),Io=p(t),te=a(t,"H2",{class:!0});var Zo=i(te);be=a(Zo,"A",{id:!0,class:!0,href:!0});var cl=i(be);_n=a(cl,"SPAN",{});var dl=i(_n);I(Re.$$.fragment,dl),dl.forEach(n),cl.forEach(n),Rr=p(Zo),vn=a(Zo,"SPAN",{});var pl=i(vn);Dr=r(pl,"InstructBlipVisionConfig"),pl.forEach(n),Zo.forEach(n),Bo=p(t),W=a(t,"DIV",{class:!0});var xe=i(W);I(De.$$.fragment,xe),Or=p(xe),ne=a(xe,"P",{});var on=i(ne);Xr=r(on,"This is the configuration class to store the configuration of a "),Wt=a(on,"A",{href:!0});var ml=i(Wt);Hr=r(ml,"InstructBlipVisionModel"),ml.forEach(n),Yr=r(on,`. It is used to
instantiate a InstructBLIP vision encoder according to the specified arguments, defining the model architecture.
Instantiating a configuration defaults will yield a similar configuration to that of the InstructBLIP
`),Oe=a(on,"A",{href:!0,rel:!0});var ul=i(Oe);Kr=r(ul,"Salesforce/instruct-blip-flan-t5"),ul.forEach(n),es=r(on," architecture."),on.forEach(n),ts=p(xe),oe=a(xe,"P",{});var rn=i(oe);ns=r(rn,"Configuration objects inherit from "),Zt=a(rn,"A",{href:!0});var fl=i(Zt);os=r(fl,"PretrainedConfig"),fl.forEach(n),rs=r(rn,` and can be used to control the model outputs. Read the
documentation from `),Qt=a(rn,"A",{href:!0});var hl=i(Qt);ss=r(hl,"PretrainedConfig"),hl.forEach(n),as=r(rn," for more information."),rn.forEach(n),is=p(xe),I(ye.$$.fragment,xe),xe.forEach(n),wo=p(t),re=a(t,"H2",{class:!0});var Qo=i(re);Ie=a(Qo,"A",{id:!0,class:!0,href:!0});var gl=i(Ie);bn=a(gl,"SPAN",{});var _l=i(bn);I(Xe.$$.fragment,_l),_l.forEach(n),gl.forEach(n),ls=p(Qo),yn=a(Qo,"SPAN",{});var vl=i(yn);cs=r(vl,"InstructBlipQFormerConfig"),vl.forEach(n),Qo.forEach(n),Mo=p(t),Z=a(t,"DIV",{class:!0});var Je=i(Z);I(He.$$.fragment,Je),ds=p(Je),Q=a(Je,"P",{});var X=i(Q);ps=r(X,"This is the configuration class to store the configuration of a "),qt=a(X,"A",{href:!0});var bl=i(qt);ms=r(bl,"InstructBlipQFormerModel"),bl.forEach(n),us=r(X,`. It is used to
instantiate a InstructBLIP Querying Transformer (Q-Former) model according to the specified arguments, defining the
model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
the InstructBLIP `),Ye=a(X,"A",{href:!0,rel:!0});var yl=i(Ye);fs=r(yl,"Salesforce/instruct-blip-flan-t5"),yl.forEach(n),hs=r(X,`
architecture. Configuration objects inherit from `),Gt=a(X,"A",{href:!0});var Il=i(Gt);gs=r(Il,"PretrainedConfig"),Il.forEach(n),_s=r(X,` and can be used to control the model outputs.
Read the documentation from `),Nt=a(X,"A",{href:!0});var Bl=i(Nt);vs=r(Bl,"PretrainedConfig"),Bl.forEach(n),bs=r(X," for more information."),X.forEach(n),ys=p(Je),se=a(Je,"P",{});var sn=i(se);Is=r(sn,"Note that "),St=a(sn,"A",{href:!0});var wl=i(St);Bs=r(wl,"InstructBlipQFormerModel"),wl.forEach(n),ws=r(sn," is very similar to "),Vt=a(sn,"A",{href:!0});var Ml=i(Vt);Ms=r(Ml,"BertLMHeadModel"),Ml.forEach(n),Ts=r(sn," with interleaved cross-attention."),sn.forEach(n),$s=p(Je),I(Be.$$.fragment,Je),Je.forEach(n),To=p(t),ae=a(t,"H2",{class:!0});var qo=i(ae);we=a(qo,"A",{id:!0,class:!0,href:!0});var Tl=i(we);In=a(Tl,"SPAN",{});var $l=i(In);I(Ke.$$.fragment,$l),$l.forEach(n),Tl.forEach(n),Cs=p(qo),Bn=a(qo,"SPAN",{});var Cl=i(Bn);ks=r(Cl,"InstructBlipProcessor"),Cl.forEach(n),qo.forEach(n),$o=p(t),P=a(t,"DIV",{class:!0});var H=i(P);I(et.$$.fragment,H),Es=p(H),wn=a(H,"P",{});var kl=i(wn);js=r(kl,`Constructs an InstructBLIP processor which wraps a BLIP image processor and a LLaMa/T5 tokenizer into a single
processor.`),kl.forEach(n),Fs=p(H),J=a(H,"P",{});var S=i(J);At=a(S,"A",{href:!0});var El=i(At);Ps=r(El,"InstructBlipProcessor"),El.forEach(n),xs=r(S," offers all the functionalities of "),Lt=a(S,"A",{href:!0});var jl=i(Lt);Js=r(jl,"BlipImageProcessor"),jl.forEach(n),zs=r(S," and "),Rt=a(S,"A",{href:!0});var Fl=i(Rt);Us=r(Fl,"AutoTokenizer"),Fl.forEach(n),Ws=r(S,`. See the
docstring of `),Mn=a(S,"CODE",{});var Pl=i(Mn);Zs=r(Pl,"__call__()"),Pl.forEach(n),Qs=r(S," and "),Dt=a(S,"A",{href:!0});var xl=i(Dt);qs=r(xl,"decode()"),xl.forEach(n),Gs=r(S," for more information."),S.forEach(n),Ns=p(H),Me=a(H,"DIV",{class:!0});var Go=i(Me);I(tt.$$.fragment,Go),Ss=p(Go),nt=a(Go,"P",{});var No=i(nt);Vs=r(No,"This method forwards all its arguments to PreTrainedTokenizer\u2019s "),Ot=a(No,"A",{href:!0});var Jl=i(Ot);As=r(Jl,"batch_decode()"),Jl.forEach(n),Ls=r(No,`. Please
refer to the docstring of this method for more information.`),No.forEach(n),Go.forEach(n),Rs=p(H),Te=a(H,"DIV",{class:!0});var So=i(Te);I(ot.$$.fragment,So),Ds=p(So),rt=a(So,"P",{});var Vo=i(rt);Os=r(Vo,"This method forwards all its arguments to PreTrainedTokenizer\u2019s "),Xt=a(Vo,"A",{href:!0});var zl=i(Xt);Xs=r(zl,"decode()"),zl.forEach(n),Hs=r(Vo,`. Please refer
to the docstring of this method for more information.`),Vo.forEach(n),So.forEach(n),H.forEach(n),Co=p(t),ie=a(t,"H2",{class:!0});var Ao=i(ie);$e=a(Ao,"A",{id:!0,class:!0,href:!0});var Ul=i($e);Tn=a(Ul,"SPAN",{});var Wl=i(Tn);I(st.$$.fragment,Wl),Wl.forEach(n),Ul.forEach(n),Ys=p(Ao),$n=a(Ao,"SPAN",{});var Zl=i($n);Ks=r(Zl,"InstructBlipVisionModel"),Zl.forEach(n),Ao.forEach(n),ko=p(t),le=a(t,"DIV",{class:!0});var Lo=i(le);I(at.$$.fragment,Lo),ea=p(Lo),R=a(Lo,"DIV",{class:!0});var an=i(R);I(it.$$.fragment,an),ta=p(an),ce=a(an,"P",{});var ln=i(ce);na=r(ln,"The "),Ht=a(ln,"A",{href:!0});var Ql=i(Ht);oa=r(Ql,"InstructBlipVisionModel"),Ql.forEach(n),ra=r(ln," forward method, overrides the "),Cn=a(ln,"CODE",{});var ql=i(Cn);sa=r(ql,"__call__"),ql.forEach(n),aa=r(ln," special method."),ln.forEach(n),ia=p(an),I(Ce.$$.fragment,an),an.forEach(n),Lo.forEach(n),Eo=p(t),de=a(t,"H2",{class:!0});var Ro=i(de);ke=a(Ro,"A",{id:!0,class:!0,href:!0});var Gl=i(ke);kn=a(Gl,"SPAN",{});var Nl=i(kn);I(lt.$$.fragment,Nl),Nl.forEach(n),Gl.forEach(n),la=p(Ro),En=a(Ro,"SPAN",{});var Sl=i(En);ca=r(Sl,"InstructBlipQFormerModel"),Sl.forEach(n),Ro.forEach(n),jo=p(t),N=a(t,"DIV",{class:!0});var cn=i(N);I(ct.$$.fragment,cn),da=p(cn),jn=a(cn,"P",{});var Vl=i(jn);pa=r(Vl,`Querying Transformer (Q-Former), used in InstructBLIP. Slightly modified from BLIP-2 as it also takes the
instruction as input.`),Vl.forEach(n),ma=p(cn),D=a(cn,"DIV",{class:!0});var dn=i(D);I(dt.$$.fragment,dn),ua=p(dn),k=a(dn,"P",{});var j=i(k);fa=r(j,"encoder_hidden_states  ("),Fn=a(j,"CODE",{});var Al=i(Fn);ha=r(Al,"torch.FloatTensor"),Al.forEach(n),ga=r(j," of shape "),Pn=a(j,"CODE",{});var Ll=i(Pn);_a=r(Ll,"(batch_size, sequence_length, hidden_size)"),Ll.forEach(n),va=r(j,", "),xn=a(j,"EM",{});var Rl=i(xn);ba=r(Rl,"optional"),Rl.forEach(n),ya=r(j,`):
Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
the model is configured as a decoder.
encoder_attention_mask (`),Jn=a(j,"CODE",{});var Dl=i(Jn);Ia=r(Dl,"torch.FloatTensor"),Dl.forEach(n),Ba=r(j," of shape "),zn=a(j,"CODE",{});var Ol=i(zn);wa=r(Ol,"(batch_size, sequence_length)"),Ol.forEach(n),Ma=r(j,", "),Un=a(j,"EM",{});var Xl=i(Un);Ta=r(Xl,"optional"),Xl.forEach(n),$a=r(j,`):
Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
the cross-attention if the model is configured as a decoder. Mask values selected in `),Wn=a(j,"CODE",{});var Hl=i(Wn);Ca=r(Hl,"[0, 1]"),Hl.forEach(n),ka=r(j,":"),j.forEach(n),Ea=p(dn),pt=a(dn,"UL",{});var Do=i(pt);mt=a(Do,"LI",{});var Oo=i(mt);ja=r(Oo,"1 for tokens that are "),Zn=a(Oo,"STRONG",{});var Yl=i(Zn);Fa=r(Yl,"not masked"),Yl.forEach(n),Pa=r(Oo,","),Oo.forEach(n),xa=p(Do),b=a(Do,"LI",{});var $=i(b);Ja=r($,"0 for tokens that are "),Qn=a($,"STRONG",{});var Kl=i(Qn);za=r(Kl,"masked"),Kl.forEach(n),Ua=r($,`.
past_key_values (`),qn=a($,"CODE",{});var ec=i(qn);Wa=r(ec,"tuple(tuple(torch.FloatTensor))"),ec.forEach(n),Za=r($," of length "),Gn=a($,"CODE",{});var tc=i(Gn);Qa=r(tc,"config.n_layers"),tc.forEach(n),qa=r($,` with each tuple having 4 tensors of:
shape `),Nn=a($,"CODE",{});var nc=i(Nn);Ga=r(nc,"(batch_size, num_heads, sequence_length - 1, embed_size_per_head)"),nc.forEach(n),Na=r($,`): Contains precomputed key and
value hidden states of the attention blocks. Can be used to speed up decoding. If `),Sn=a($,"CODE",{});var oc=i(Sn);Sa=r(oc,"past_key_values"),oc.forEach(n),Va=r($,` are
used, the user can optionally input only the last `),Vn=a($,"CODE",{});var rc=i(Vn);Aa=r(rc,"decoder_input_ids"),rc.forEach(n),La=r($,` (those that don\u2019t have their past key
value states given to this model) of shape `),An=a($,"CODE",{});var sc=i(An);Ra=r(sc,"(batch_size, 1)"),sc.forEach(n),Da=r($," instead of all "),Ln=a($,"CODE",{});var ac=i(Ln);Oa=r(ac,"decoder_input_ids"),ac.forEach(n),Xa=r($,` of shape
`),Rn=a($,"CODE",{});var ic=i(Rn);Ha=r(ic,"(batch_size, sequence_length)"),ic.forEach(n),Ya=r($,`.
use_cache (`),Dn=a($,"CODE",{});var lc=i(Dn);Ka=r(lc,"bool"),lc.forEach(n),ei=r($,", "),On=a($,"EM",{});var cc=i(On);ti=r(cc,"optional"),cc.forEach(n),ni=r($,`):
If set to `),Xn=a($,"CODE",{});var dc=i(Xn);oi=r(dc,"True"),dc.forEach(n),ri=r($,", "),Hn=a($,"CODE",{});var pc=i(Hn);si=r(pc,"past_key_values"),pc.forEach(n),ai=r($,` key value states are returned and can be used to speed up decoding (see
`),Yn=a($,"CODE",{});var mc=i(Yn);ii=r(mc,"past_key_values"),mc.forEach(n),li=r($,")."),$.forEach(n),Do.forEach(n),dn.forEach(n),cn.forEach(n),Fo=p(t),pe=a(t,"H2",{class:!0});var Xo=i(pe);Ee=a(Xo,"A",{id:!0,class:!0,href:!0});var uc=i(Ee);Kn=a(uc,"SPAN",{});var fc=i(Kn);I(ut.$$.fragment,fc),fc.forEach(n),uc.forEach(n),ci=p(Xo),eo=a(Xo,"SPAN",{});var hc=i(eo);di=r(hc,"InstructBlipForConditionalGeneration"),hc.forEach(n),Xo.forEach(n),Po=p(t),E=a(t,"DIV",{class:!0});var z=i(E);I(ft.$$.fragment,z),pi=p(z),to=a(z,"P",{});var gc=i(to);mi=r(gc,`InstructBLIP Model for generating text given an image and an optional text prompt. The model consists of a vision
encoder, Querying Transformer (Q-Former) and a language model.`),gc.forEach(n),ui=p(z),ht=a(z,"P",{});var Ho=i(ht);fi=r(Ho,"One can optionally pass "),no=a(Ho,"CODE",{});var _c=i(no);hi=r(_c,"input_ids"),_c.forEach(n),gi=r(Ho,` to the model, which serve as a text prompt, to make the language model continue
the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.`),Ho.forEach(n),_i=p(z),gt=a(z,"P",{});var Yo=i(gt);vi=r(Yo,"This model inherits from "),Yt=a(Yo,"A",{href:!0});var vc=i(Yt);bi=r(vc,"PreTrainedModel"),vc.forEach(n),yi=r(Yo,`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),Yo.forEach(n),Ii=p(z),_t=a(z,"P",{});var Ko=i(_t);Bi=r(Ko,"This model is also a PyTorch "),vt=a(Ko,"A",{href:!0,rel:!0});var bc=i(vt);wi=r(bc,"torch.nn.Module"),bc.forEach(n),Mi=r(Ko,` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),Ko.forEach(n),Ti=p(z),q=a(z,"DIV",{class:!0});var ze=i(q);I(bt.$$.fragment,ze),$i=p(ze),me=a(ze,"P",{});var pn=i(me);Ci=r(pn,"The "),Kt=a(pn,"A",{href:!0});var yc=i(Kt);ki=r(yc,"InstructBlipForConditionalGeneration"),yc.forEach(n),Ei=r(pn," forward method, overrides the "),oo=a(pn,"CODE",{});var Ic=i(oo);ji=r(Ic,"__call__"),Ic.forEach(n),Fi=r(pn," special method."),pn.forEach(n),Pi=p(ze),I(je.$$.fragment,ze),xi=p(ze),I(Fe.$$.fragment,ze),ze.forEach(n),Ji=p(z),Pe=a(z,"DIV",{class:!0});var er=i(Pe);I(yt.$$.fragment,er),zi=p(er),It=a(er,"P",{});var tr=i(It);Ui=r(tr,"Overrides "),ro=a(tr,"CODE",{});var Bc=i(ro);Wi=r(Bc,"generate"),Bc.forEach(n),Zi=r(tr," function to be able to use the model as a conditional generator."),tr.forEach(n),er.forEach(n),z.forEach(n),this.h()},h(){l(f,"name","hf:doc:metadata"),l(f,"content",JSON.stringify(Wc)),l(h,"id","instructblip"),l(h,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(h,"href","#instructblip"),l(_,"class","relative group"),l(fe,"id","overview"),l(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(fe,"href","#overview"),l(Y,"class","relative group"),l(We,"href","https://arxiv.org/abs/2305.06500"),l(We,"rel","nofollow"),l(Tt,"href","blip2"),l(jt,"href","blip2"),kc(he.src,qi="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/instructblip_architecture.jpg")||l(he,"src",qi),l(he,"alt","drawing"),l(he,"width","600"),l(Ft,"href","https://arxiv.org/abs/2305.06500"),l(qe,"href","https://huggingface.co/nielsr"),l(qe,"rel","nofollow"),l(Ge,"href","https://github.com/salesforce/LAVIS/tree/main/projects/instructblip"),l(Ge,"rel","nofollow"),l(ge,"id","transformers.InstructBlipConfig"),l(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(ge,"href","#transformers.InstructBlipConfig"),l(K,"class","relative group"),l(Pt,"href","/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipConfig"),l(xt,"href","/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipForConditionalGeneration"),l(Ve,"href","https://huggingface.co/Salesforce/instruct-blip-flan-t5"),l(Ve,"rel","nofollow"),l(Jt,"href","/docs/transformers/v4.33.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(zt,"href","/docs/transformers/v4.33.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(Ut,"href","/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipConfig"),l(ve,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(be,"id","transformers.InstructBlipVisionConfig"),l(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(be,"href","#transformers.InstructBlipVisionConfig"),l(te,"class","relative group"),l(Wt,"href","/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipVisionModel"),l(Oe,"href","https://huggingface.co/Salesforce/instruct-blip-flan-t5"),l(Oe,"rel","nofollow"),l(Zt,"href","/docs/transformers/v4.33.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(Qt,"href","/docs/transformers/v4.33.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(Ie,"id","transformers.InstructBlipQFormerConfig"),l(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(Ie,"href","#transformers.InstructBlipQFormerConfig"),l(re,"class","relative group"),l(qt,"href","/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipQFormerModel"),l(Ye,"href","https://huggingface.co/Salesforce/instruct-blip-flan-t5"),l(Ye,"rel","nofollow"),l(Gt,"href","/docs/transformers/v4.33.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(Nt,"href","/docs/transformers/v4.33.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(St,"href","/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipQFormerModel"),l(Vt,"href","/docs/transformers/v4.33.0/en/model_doc/bert#transformers.BertLMHeadModel"),l(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(we,"id","transformers.InstructBlipProcessor"),l(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(we,"href","#transformers.InstructBlipProcessor"),l(ae,"class","relative group"),l(At,"href","/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipProcessor"),l(Lt,"href","/docs/transformers/v4.33.0/en/model_doc/blip#transformers.BlipImageProcessor"),l(Rt,"href","/docs/transformers/v4.33.0/en/model_doc/auto#transformers.AutoTokenizer"),l(Dt,"href","/docs/transformers/v4.33.0/en/model_doc/blip#transformers.BlipProcessor.decode"),l(Ot,"href","/docs/transformers/v4.33.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.batch_decode"),l(Me,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(Xt,"href","/docs/transformers/v4.33.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.decode"),l(Te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l($e,"id","transformers.InstructBlipVisionModel"),l($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l($e,"href","#transformers.InstructBlipVisionModel"),l(ie,"class","relative group"),l(Ht,"href","/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipVisionModel"),l(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(ke,"id","transformers.InstructBlipQFormerModel"),l(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(ke,"href","#transformers.InstructBlipQFormerModel"),l(de,"class","relative group"),l(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(Ee,"id","transformers.InstructBlipForConditionalGeneration"),l(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(Ee,"href","#transformers.InstructBlipForConditionalGeneration"),l(pe,"class","relative group"),l(Yt,"href","/docs/transformers/v4.33.0/en/main_classes/model#transformers.PreTrainedModel"),l(vt,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),l(vt,"rel","nofollow"),l(Kt,"href","/docs/transformers/v4.33.0/en/model_doc/instructblip#transformers.InstructBlipForConditionalGeneration"),l(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(Pe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,m){e(document.head,f),u(t,C,m),u(t,_,m),e(_,h),e(h,v),B(c,v,null),e(_,g),e(_,G),e(G,sr),u(t,co,m),u(t,Y,m),e(Y,fe),e(fe,mn),B(Ue,mn,null),e(Y,ar),e(Y,un),e(un,ir),u(t,po,m),u(t,V,m),e(V,lr),e(V,We),e(We,cr),e(V,dr),e(V,Tt),e(Tt,pr),e(V,mr),u(t,mo,m),u(t,$t,m),e($t,ur),u(t,uo,m),u(t,Ct,m),e(Ct,fn),e(fn,fr),u(t,fo,m),u(t,kt,m),e(kt,hr),u(t,ho,m),u(t,Et,m),e(Et,Ze),e(Ze,gr),e(Ze,jt),e(jt,_r),e(Ze,vr),u(t,go,m),u(t,he,m),u(t,_o,m),u(t,Qe,m),e(Qe,br),e(Qe,Ft),e(Ft,yr),u(t,vo,m),u(t,A,m),e(A,Ir),e(A,qe),e(qe,Br),e(A,wr),e(A,Ge),e(Ge,Mr),e(A,Tr),u(t,bo,m),u(t,K,m),e(K,ge),e(ge,hn),B(Ne,hn,null),e(K,$r),e(K,gn),e(gn,Cr),u(t,yo,m),u(t,F,m),B(Se,F,null),e(F,kr),e(F,L),e(L,Pt),e(Pt,Er),e(L,jr),e(L,xt),e(xt,Fr),e(L,Pr),e(L,Ve),e(Ve,xr),e(L,Jr),e(F,zr),e(F,ee),e(ee,Ur),e(ee,Jt),e(Jt,Wr),e(ee,Zr),e(ee,zt),e(zt,Qr),e(ee,qr),e(F,Gr),B(_e,F,null),e(F,Nr),e(F,ve),B(Ae,ve,null),e(ve,Sr),e(ve,Le),e(Le,Vr),e(Le,Ut),e(Ut,Ar),e(Le,Lr),u(t,Io,m),u(t,te,m),e(te,be),e(be,_n),B(Re,_n,null),e(te,Rr),e(te,vn),e(vn,Dr),u(t,Bo,m),u(t,W,m),B(De,W,null),e(W,Or),e(W,ne),e(ne,Xr),e(ne,Wt),e(Wt,Hr),e(ne,Yr),e(ne,Oe),e(Oe,Kr),e(ne,es),e(W,ts),e(W,oe),e(oe,ns),e(oe,Zt),e(Zt,os),e(oe,rs),e(oe,Qt),e(Qt,ss),e(oe,as),e(W,is),B(ye,W,null),u(t,wo,m),u(t,re,m),e(re,Ie),e(Ie,bn),B(Xe,bn,null),e(re,ls),e(re,yn),e(yn,cs),u(t,Mo,m),u(t,Z,m),B(He,Z,null),e(Z,ds),e(Z,Q),e(Q,ps),e(Q,qt),e(qt,ms),e(Q,us),e(Q,Ye),e(Ye,fs),e(Q,hs),e(Q,Gt),e(Gt,gs),e(Q,_s),e(Q,Nt),e(Nt,vs),e(Q,bs),e(Z,ys),e(Z,se),e(se,Is),e(se,St),e(St,Bs),e(se,ws),e(se,Vt),e(Vt,Ms),e(se,Ts),e(Z,$s),B(Be,Z,null),u(t,To,m),u(t,ae,m),e(ae,we),e(we,In),B(Ke,In,null),e(ae,Cs),e(ae,Bn),e(Bn,ks),u(t,$o,m),u(t,P,m),B(et,P,null),e(P,Es),e(P,wn),e(wn,js),e(P,Fs),e(P,J),e(J,At),e(At,Ps),e(J,xs),e(J,Lt),e(Lt,Js),e(J,zs),e(J,Rt),e(Rt,Us),e(J,Ws),e(J,Mn),e(Mn,Zs),e(J,Qs),e(J,Dt),e(Dt,qs),e(J,Gs),e(P,Ns),e(P,Me),B(tt,Me,null),e(Me,Ss),e(Me,nt),e(nt,Vs),e(nt,Ot),e(Ot,As),e(nt,Ls),e(P,Rs),e(P,Te),B(ot,Te,null),e(Te,Ds),e(Te,rt),e(rt,Os),e(rt,Xt),e(Xt,Xs),e(rt,Hs),u(t,Co,m),u(t,ie,m),e(ie,$e),e($e,Tn),B(st,Tn,null),e(ie,Ys),e(ie,$n),e($n,Ks),u(t,ko,m),u(t,le,m),B(at,le,null),e(le,ea),e(le,R),B(it,R,null),e(R,ta),e(R,ce),e(ce,na),e(ce,Ht),e(Ht,oa),e(ce,ra),e(ce,Cn),e(Cn,sa),e(ce,aa),e(R,ia),B(Ce,R,null),u(t,Eo,m),u(t,de,m),e(de,ke),e(ke,kn),B(lt,kn,null),e(de,la),e(de,En),e(En,ca),u(t,jo,m),u(t,N,m),B(ct,N,null),e(N,da),e(N,jn),e(jn,pa),e(N,ma),e(N,D),B(dt,D,null),e(D,ua),e(D,k),e(k,fa),e(k,Fn),e(Fn,ha),e(k,ga),e(k,Pn),e(Pn,_a),e(k,va),e(k,xn),e(xn,ba),e(k,ya),e(k,Jn),e(Jn,Ia),e(k,Ba),e(k,zn),e(zn,wa),e(k,Ma),e(k,Un),e(Un,Ta),e(k,$a),e(k,Wn),e(Wn,Ca),e(k,ka),e(D,Ea),e(D,pt),e(pt,mt),e(mt,ja),e(mt,Zn),e(Zn,Fa),e(mt,Pa),e(pt,xa),e(pt,b),e(b,Ja),e(b,Qn),e(Qn,za),e(b,Ua),e(b,qn),e(qn,Wa),e(b,Za),e(b,Gn),e(Gn,Qa),e(b,qa),e(b,Nn),e(Nn,Ga),e(b,Na),e(b,Sn),e(Sn,Sa),e(b,Va),e(b,Vn),e(Vn,Aa),e(b,La),e(b,An),e(An,Ra),e(b,Da),e(b,Ln),e(Ln,Oa),e(b,Xa),e(b,Rn),e(Rn,Ha),e(b,Ya),e(b,Dn),e(Dn,Ka),e(b,ei),e(b,On),e(On,ti),e(b,ni),e(b,Xn),e(Xn,oi),e(b,ri),e(b,Hn),e(Hn,si),e(b,ai),e(b,Yn),e(Yn,ii),e(b,li),u(t,Fo,m),u(t,pe,m),e(pe,Ee),e(Ee,Kn),B(ut,Kn,null),e(pe,ci),e(pe,eo),e(eo,di),u(t,Po,m),u(t,E,m),B(ft,E,null),e(E,pi),e(E,to),e(to,mi),e(E,ui),e(E,ht),e(ht,fi),e(ht,no),e(no,hi),e(ht,gi),e(E,_i),e(E,gt),e(gt,vi),e(gt,Yt),e(Yt,bi),e(gt,yi),e(E,Ii),e(E,_t),e(_t,Bi),e(_t,vt),e(vt,wi),e(_t,Mi),e(E,Ti),e(E,q),B(bt,q,null),e(q,$i),e(q,me),e(me,Ci),e(me,Kt),e(Kt,ki),e(me,Ei),e(me,oo),e(oo,ji),e(me,Fi),e(q,Pi),B(je,q,null),e(q,xi),B(Fe,q,null),e(E,Ji),e(E,Pe),B(yt,Pe,null),e(Pe,zi),e(Pe,It),e(It,Ui),e(It,ro),e(ro,Wi),e(It,Zi),xo=!0},p(t,[m]){const Bt={};m&2&&(Bt.$$scope={dirty:m,ctx:t}),_e.$set(Bt);const so={};m&2&&(so.$$scope={dirty:m,ctx:t}),ye.$set(so);const ao={};m&2&&(ao.$$scope={dirty:m,ctx:t}),Be.$set(ao);const io={};m&2&&(io.$$scope={dirty:m,ctx:t}),Ce.$set(io);const wt={};m&2&&(wt.$$scope={dirty:m,ctx:t}),je.$set(wt);const lo={};m&2&&(lo.$$scope={dirty:m,ctx:t}),Fe.$set(lo)},i(t){xo||(w(c.$$.fragment,t),w(Ue.$$.fragment,t),w(Ne.$$.fragment,t),w(Se.$$.fragment,t),w(_e.$$.fragment,t),w(Ae.$$.fragment,t),w(Re.$$.fragment,t),w(De.$$.fragment,t),w(ye.$$.fragment,t),w(Xe.$$.fragment,t),w(He.$$.fragment,t),w(Be.$$.fragment,t),w(Ke.$$.fragment,t),w(et.$$.fragment,t),w(tt.$$.fragment,t),w(ot.$$.fragment,t),w(st.$$.fragment,t),w(at.$$.fragment,t),w(it.$$.fragment,t),w(Ce.$$.fragment,t),w(lt.$$.fragment,t),w(ct.$$.fragment,t),w(dt.$$.fragment,t),w(ut.$$.fragment,t),w(ft.$$.fragment,t),w(bt.$$.fragment,t),w(je.$$.fragment,t),w(Fe.$$.fragment,t),w(yt.$$.fragment,t),xo=!0)},o(t){M(c.$$.fragment,t),M(Ue.$$.fragment,t),M(Ne.$$.fragment,t),M(Se.$$.fragment,t),M(_e.$$.fragment,t),M(Ae.$$.fragment,t),M(Re.$$.fragment,t),M(De.$$.fragment,t),M(ye.$$.fragment,t),M(Xe.$$.fragment,t),M(He.$$.fragment,t),M(Be.$$.fragment,t),M(Ke.$$.fragment,t),M(et.$$.fragment,t),M(tt.$$.fragment,t),M(ot.$$.fragment,t),M(st.$$.fragment,t),M(at.$$.fragment,t),M(it.$$.fragment,t),M(Ce.$$.fragment,t),M(lt.$$.fragment,t),M(ct.$$.fragment,t),M(dt.$$.fragment,t),M(ut.$$.fragment,t),M(ft.$$.fragment,t),M(bt.$$.fragment,t),M(je.$$.fragment,t),M(Fe.$$.fragment,t),M(yt.$$.fragment,t),xo=!1},d(t){n(f),t&&n(C),t&&n(_),T(c),t&&n(co),t&&n(Y),T(Ue),t&&n(po),t&&n(V),t&&n(mo),t&&n($t),t&&n(uo),t&&n(Ct),t&&n(fo),t&&n(kt),t&&n(ho),t&&n(Et),t&&n(go),t&&n(he),t&&n(_o),t&&n(Qe),t&&n(vo),t&&n(A),t&&n(bo),t&&n(K),T(Ne),t&&n(yo),t&&n(F),T(Se),T(_e),T(Ae),t&&n(Io),t&&n(te),T(Re),t&&n(Bo),t&&n(W),T(De),T(ye),t&&n(wo),t&&n(re),T(Xe),t&&n(Mo),t&&n(Z),T(He),T(Be),t&&n(To),t&&n(ae),T(Ke),t&&n($o),t&&n(P),T(et),T(tt),T(ot),t&&n(Co),t&&n(ie),T(st),t&&n(ko),t&&n(le),T(at),T(it),T(Ce),t&&n(Eo),t&&n(de),T(lt),t&&n(jo),t&&n(N),T(ct),T(dt),t&&n(Fo),t&&n(pe),T(ut),t&&n(Po),t&&n(E),T(ft),T(bt),T(je),T(Fe),T(yt)}}}const Wc={local:"instructblip",sections:[{local:"overview",title:"Overview"},{local:"transformers.InstructBlipConfig",title:"InstructBlipConfig"},{local:"transformers.InstructBlipVisionConfig",title:"InstructBlipVisionConfig"},{local:"transformers.InstructBlipQFormerConfig",title:"InstructBlipQFormerConfig"},{local:"transformers.InstructBlipProcessor",title:"InstructBlipProcessor"},{local:"transformers.InstructBlipVisionModel",title:"InstructBlipVisionModel"},{local:"transformers.InstructBlipQFormerModel",title:"InstructBlipQFormerModel"},{local:"transformers.InstructBlipForConditionalGeneration",title:"InstructBlipForConditionalGeneration"}],title:"InstructBLIP"};function Zc(x){return Ec(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ac extends Mc{constructor(f){super();Tc(this,f,Zc,Uc,$c,{})}}export{Ac as default,Wc as metadata};
