import{S as os,i as rs,s as ns,e as r,k as d,w as f,t as a,M as ls,c as n,d as o,m as p,a as l,x as m,h as s,b as c,G as t,g as h,y as u,q as w,o as y,B as v,v as as}from"../chunks/vendor-hf-doc-builder.js";import{T as ss}from"../chunks/Tip-hf-doc-builder.js";import{I as S}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as ct}from"../chunks/CodeBlock-hf-doc-builder.js";function is(to){let b,H;return{c(){b=r("p"),H=a(`This is the very beginning of our experiments with TorchScript and we are still
exploring its capabilities with variable-input-size models. It is a focus of interest to
us and we will deepen our analysis in upcoming releases, with more code examples, a more
flexible implementation, and benchmarks comparing Python-based codes with compiled
TorchScript.`)},l(g){b=n(g,"P",{});var T=l(b);H=s(T,`This is the very beginning of our experiments with TorchScript and we are still
exploring its capabilities with variable-input-size models. It is a focus of interest to
us and we will deepen our analysis in upcoming releases, with more code examples, a more
flexible implementation, and benchmarks comparing Python-based codes with compiled
TorchScript.`),T.forEach(o)},m(g,T){h(g,b,T),t(b,H)},d(g){g&&o(b)}}}function hs(to){let b,H,g,T,dt,me,pr,pt,fr,oo,X,ro,D,mr,ue,ur,wr,no,Ye,ft,yr,lo,F,vr,we,br,Mr,ao,qe,gr,so,Ke,Er,io,Y,ye,Tr,mt,_r,Ur,Jr,ut,jr,ho,Oe,$r,co,N,q,wt,ve,Sr,yt,kr,po,_,Wr,vt,Ar,Ir,bt,Nr,Br,Mt,Cr,Zr,fo,U,xr,gt,zr,Rr,Et,Pr,Qr,Tt,Vr,Gr,mo,K,Hr,_t,Xr,Dr,uo,B,O,Ut,be,Fr,Jt,Yr,wo,L,qr,jt,Kr,Or,yo,Le,Lr,vo,Me,bo,et,en,Mo,tt,tn,go,C,ee,$t,ge,on,St,rn,Eo,ot,nn,To,Z,te,kt,Ee,ln,Wt,an,_o,E,sn,At,hn,cn,It,dn,pn,Nt,fn,mn,Bt,un,wn,Uo,Te,Jo,x,oe,Ct,_e,yn,Zt,vn,jo,J,bn,xt,Mn,gn,zt,En,Tn,Rt,_n,Un,$o,Ue,So,z,re,Pt,Je,Jn,Qt,jn,ko,ne,$n,Vt,Sn,kn,Wo,je,Ao,R,le,Gt,$e,Wn,Ht,An,Io,k,In,Se,Nn,Bn,ke,Cn,Zn,No,W,Xt,xn,zn,We,Rn,Ae,Pn,Qn,Vn,P,Gn,Ie,Hn,Xn,Ne,Dn,Fn,Bo,Q,ae,Dt,Be,Yn,Ft,qn,Co,M,Kn,Ce,On,Ln,Ze,el,tl,xe,ol,rl,ze,nl,ll,Re,al,sl,Zo,V,se,Yt,Pe,il,qt,hl,xo,A,cl,Qe,dl,pl,Ve,fl,ml,zo,G,ie,Kt,Ge,ul,Ot,wl,Ro,j,yl,rt,vl,bl,Lt,Ml,gl,eo,El,Tl,Po,He,Qo,nt,_l,Vo,Xe,Go,lt,Ul,Ho,he,Jl,De,jl,$l,Xo;return me=new S({}),X=new ss({props:{$$slots:{default:[is]},$$scope:{ctx:to}}}),ve=new S({}),be=new S({}),Me=new ct({props:{code:"JTYwVGhlJTIwZXhwYW5kZWQlMjBzaXplJTIwb2YlMjB0aGUlMjB0ZW5zb3IlMjAoMyklMjBtdXN0JTIwbWF0Y2glMjB0aGUlMjBleGlzdGluZyUyMHNpemUlMjAoNyklMjBhdCUyMG5vbi1zaW5nbGV0b24lMjBkaW1lbnNpb24lMjAyJTYw",highlighted:'`The expanded <span class="hljs-built_in">size</span> of the tensor (<span class="hljs-number">3</span>) must match the existing <span class="hljs-built_in">size</span> (<span class="hljs-number">7</span>) at non-singleton <span class="hljs-keyword">dimension</span> <span class="hljs-number">2</span>`'}}),ge=new S({}),Ee=new S({}),Te=new ct({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJlcnRNb2RlbCUyQyUyMEJlcnRUb2tlbml6ZXIlMkMlMjBCZXJ0Q29uZmlnJTBBaW1wb3J0JTIwdG9yY2glMEElMEFlbmMlMjAlM0QlMjBCZXJ0VG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtdW5jYXNlZCUyMiklMEElMEElMjMlMjBUb2tlbml6aW5nJTIwaW5wdXQlMjB0ZXh0JTBBdGV4dCUyMCUzRCUyMCUyMiU1QkNMUyU1RCUyMFdobyUyMHdhcyUyMEppbSUyMEhlbnNvbiUyMCUzRiUyMCU1QlNFUCU1RCUyMEppbSUyMEhlbnNvbiUyMHdhcyUyMGElMjBwdXBwZXRlZXIlMjAlNUJTRVAlNUQlMjIlMEF0b2tlbml6ZWRfdGV4dCUyMCUzRCUyMGVuYy50b2tlbml6ZSh0ZXh0KSUwQSUwQSUyMyUyME1hc2tpbmclMjBvbmUlMjBvZiUyMHRoZSUyMGlucHV0JTIwdG9rZW5zJTBBbWFza2VkX2luZGV4JTIwJTNEJTIwOCUwQXRva2VuaXplZF90ZXh0JTVCbWFza2VkX2luZGV4JTVEJTIwJTNEJTIwJTIyJTVCTUFTSyU1RCUyMiUwQWluZGV4ZWRfdG9rZW5zJTIwJTNEJTIwZW5jLmNvbnZlcnRfdG9rZW5zX3RvX2lkcyh0b2tlbml6ZWRfdGV4dCklMEFzZWdtZW50c19pZHMlMjAlM0QlMjAlNUIwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSU1RCUwQSUwQSUyMyUyMENyZWF0aW5nJTIwYSUyMGR1bW15JTIwaW5wdXQlMEF0b2tlbnNfdGVuc29yJTIwJTNEJTIwdG9yY2gudGVuc29yKCU1QmluZGV4ZWRfdG9rZW5zJTVEKSUwQXNlZ21lbnRzX3RlbnNvcnMlMjAlM0QlMjB0b3JjaC50ZW5zb3IoJTVCc2VnbWVudHNfaWRzJTVEKSUwQWR1bW15X2lucHV0JTIwJTNEJTIwJTVCdG9rZW5zX3RlbnNvciUyQyUyMHNlZ21lbnRzX3RlbnNvcnMlNUQlMEElMEElMjMlMjBJbml0aWFsaXppbmclMjB0aGUlMjBtb2RlbCUyMHdpdGglMjB0aGUlMjB0b3JjaHNjcmlwdCUyMGZsYWclMEElMjMlMjBGbGFnJTIwc2V0JTIwdG8lMjBUcnVlJTIwZXZlbiUyMHRob3VnaCUyMGl0JTIwaXMlMjBub3QlMjBuZWNlc3NhcnklMjBhcyUyMHRoaXMlMjBtb2RlbCUyMGRvZXMlMjBub3QlMjBoYXZlJTIwYW4lMjBMTSUyMEhlYWQuJTBBY29uZmlnJTIwJTNEJTIwQmVydENvbmZpZyglMEElMjAlMjAlMjAlMjB2b2NhYl9zaXplX29yX2NvbmZpZ19qc29uX2ZpbGUlM0QzMjAwMCUyQyUwQSUyMCUyMCUyMCUyMGhpZGRlbl9zaXplJTNENzY4JTJDJTBBJTIwJTIwJTIwJTIwbnVtX2hpZGRlbl9sYXllcnMlM0QxMiUyQyUwQSUyMCUyMCUyMCUyMG51bV9hdHRlbnRpb25faGVhZHMlM0QxMiUyQyUwQSUyMCUyMCUyMCUyMGludGVybWVkaWF0ZV9zaXplJTNEMzA3MiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoc2NyaXB0JTNEVHJ1ZSUyQyUwQSklMEElMEElMjMlMjBJbnN0YW50aWF0aW5nJTIwdGhlJTIwbW9kZWwlMEFtb2RlbCUyMCUzRCUyMEJlcnRNb2RlbChjb25maWcpJTBBJTBBJTIzJTIwVGhlJTIwbW9kZWwlMjBuZWVkcyUyMHRvJTIwYmUlMjBpbiUyMGV2YWx1YXRpb24lMjBtb2RlJTBBbW9kZWwuZXZhbCgpJTBBJTBBJTIzJTIwSWYlMjB5b3UlMjBhcmUlMjBpbnN0YW50aWF0aW5nJTIwdGhlJTIwbW9kZWwlMjB3aXRoJTIwKmZyb21fcHJldHJhaW5lZColMjB5b3UlMjBjYW4lMjBhbHNvJTIwZWFzaWx5JTIwc2V0JTIwdGhlJTIwVG9yY2hTY3JpcHQlMjBmbGFnJTBBbW9kZWwlMjAlM0QlMjBCZXJ0TW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMmJlcnQtYmFzZS11bmNhc2VkJTIyJTJDJTIwdG9yY2hzY3JpcHQlM0RUcnVlKSUwQSUwQSUyMyUyMENyZWF0aW5nJTIwdGhlJTIwdHJhY2UlMEF0cmFjZWRfbW9kZWwlMjAlM0QlMjB0b3JjaC5qaXQudHJhY2UobW9kZWwlMkMlMjAlNUJ0b2tlbnNfdGVuc29yJTJDJTIwc2VnbWVudHNfdGVuc29ycyU1RCklMEF0b3JjaC5qaXQuc2F2ZSh0cmFjZWRfbW9kZWwlMkMlMjAlMjJ0cmFjZWRfYmVydC5wdCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertModel, BertTokenizer, BertConfig
<span class="hljs-keyword">import</span> torch

enc = BertTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)

<span class="hljs-comment"># Tokenizing input text</span>
text = <span class="hljs-string">&quot;[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]&quot;</span>
tokenized_text = enc.tokenize(text)

<span class="hljs-comment"># Masking one of the input tokens</span>
masked_index = <span class="hljs-number">8</span>
tokenized_text[masked_index] = <span class="hljs-string">&quot;[MASK]&quot;</span>
indexed_tokens = enc.convert_tokens_to_ids(tokenized_text)
segments_ids = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]

<span class="hljs-comment"># Creating a dummy input</span>
tokens_tensor = torch.tensor([indexed_tokens])
segments_tensors = torch.tensor([segments_ids])
dummy_input = [tokens_tensor, segments_tensors]

<span class="hljs-comment"># Initializing the model with the torchscript flag</span>
<span class="hljs-comment"># Flag set to True even though it is not necessary as this model does not have an LM Head.</span>
config = BertConfig(
    vocab_size_or_config_json_file=<span class="hljs-number">32000</span>,
    hidden_size=<span class="hljs-number">768</span>,
    num_hidden_layers=<span class="hljs-number">12</span>,
    num_attention_heads=<span class="hljs-number">12</span>,
    intermediate_size=<span class="hljs-number">3072</span>,
    torchscript=<span class="hljs-literal">True</span>,
)

<span class="hljs-comment"># Instantiating the model</span>
model = BertModel(config)

<span class="hljs-comment"># The model needs to be in evaluation mode</span>
model.<span class="hljs-built_in">eval</span>()

<span class="hljs-comment"># If you are instantiating the model with *from_pretrained* you can also easily set the TorchScript flag</span>
model = BertModel.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>, torchscript=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Creating the trace</span>
traced_model = torch.jit.trace(model, [tokens_tensor, segments_tensors])
torch.jit.save(traced_model, <span class="hljs-string">&quot;traced_bert.pt&quot;</span>)`}}),_e=new S({}),Ue=new ct({props:{code:"bG9hZGVkX21vZGVsJTIwJTNEJTIwdG9yY2guaml0LmxvYWQoJTIydHJhY2VkX2JlcnQucHQlMjIpJTBBbG9hZGVkX21vZGVsLmV2YWwoKSUwQSUwQWFsbF9lbmNvZGVyX2xheWVycyUyQyUyMHBvb2xlZF9vdXRwdXQlMjAlM0QlMjBsb2FkZWRfbW9kZWwoKmR1bW15X2lucHV0KQ==",highlighted:`loaded_model = torch.jit.load(<span class="hljs-string">&quot;traced_bert.pt&quot;</span>)
loaded_model.<span class="hljs-built_in">eval</span>()

all_encoder_layers, pooled_output = loaded_model(*dummy_input)`}}),Je=new S({}),je=new ct({props:{code:"dHJhY2VkX21vZGVsKHRva2Vuc190ZW5zb3IlMkMlMjBzZWdtZW50c190ZW5zb3JzKQ==",highlighted:"traced_model(tokens_tensor, segments_tensors)"}}),$e=new S({}),Be=new S({}),Pe=new S({}),Ge=new S({}),He=new ct({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJlcnRNb2RlbCUyQyUyMEJlcnRUb2tlbml6ZXIlMkMlMjBCZXJ0Q29uZmlnJTBBaW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0b3JjaC5uZXVyb24=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertModel, BertTokenizer, BertConfig
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.neuron`}}),Xe=new ct({props:{code:"LSUyMHRvcmNoLmppdC50cmFjZShtb2RlbCUyQyUyMCU1QnRva2Vuc190ZW5zb3IlMkMlMjBzZWdtZW50c190ZW5zb3JzJTVEKSUwQSUyQiUyMHRvcmNoLm5ldXJvbi50cmFjZShtb2RlbCUyQyUyMCU1QnRva2VuX3RlbnNvciUyQyUyMHNlZ21lbnRzX3RlbnNvcnMlNUQp",highlighted:`<span class="hljs-deletion">- torch.jit.trace(model, [tokens_tensor, segments_tensors])</span>
<span class="hljs-addition">+ torch.neuron.trace(model, [token_tensor, segments_tensors])</span>`}}),{c(){b=r("meta"),H=d(),g=r("h1"),T=r("a"),dt=r("span"),f(me.$$.fragment),pr=d(),pt=r("span"),fr=a("Export to TorchScript"),oo=d(),f(X.$$.fragment),ro=d(),D=r("p"),mr=a("According to the "),ue=r("a"),ur=a("TorchScript documentation"),wr=a(":"),no=d(),Ye=r("blockquote"),ft=r("p"),yr=a("TorchScript is a way to create serializable and optimizable models from PyTorch code."),lo=d(),F=r("p"),vr=a("There are two PyTorch modules, "),we=r("a"),br=a(`JIT and
TRACE`),Mr=a(`, that allow developers to export their
models to be reused in other programs like efficiency-oriented C++ programs.`),ao=d(),qe=r("p"),gr=a(`We provide an interface that allows you to export \u{1F917} Transformers models to TorchScript
so they can be reused in a different environment than PyTorch-based Python programs.
Here, we explain how to export and use our models using TorchScript.`),so=d(),Ke=r("p"),Er=a("Exporting a model requires two things:"),io=d(),Y=r("ul"),ye=r("li"),Tr=a("model instantiation with the "),mt=r("code"),_r=a("torchscript"),Ur=a(" flag"),Jr=d(),ut=r("li"),jr=a("a forward pass with dummy inputs"),ho=d(),Oe=r("p"),$r=a(`These necessities imply several things developers should be careful about as detailed
below.`),co=d(),N=r("h2"),q=r("a"),wt=r("span"),f(ve.$$.fragment),Sr=d(),yt=r("span"),kr=a("TorchScript flag and tied weights"),po=d(),_=r("p"),Wr=a("The "),vt=r("code"),Ar=a("torchscript"),Ir=a(` flag is necessary because most of the \u{1F917} Transformers language models
have tied weights between their `),bt=r("code"),Nr=a("Embedding"),Br=a(" layer and their "),Mt=r("code"),Cr=a("Decoding"),Zr=a(` layer.
TorchScript does not allow you to export models that have tied weights, so it is
necessary to untie and clone the weights beforehand.`),fo=d(),U=r("p"),xr=a("Models instantiated with the "),gt=r("code"),zr=a("torchscript"),Rr=a(" flag have their "),Et=r("code"),Pr=a("Embedding"),Qr=a(` layer and
`),Tt=r("code"),Vr=a("Decoding"),Gr=a(` layer separated, which means that they should not be trained down the line.
Training would desynchronize the two layers, leading to unexpected results.`),mo=d(),K=r("p"),Hr=a(`This is not the case for models that do not have a language model head, as those do not
have tied weights. These models can be safely exported without the `),_t=r("code"),Xr=a("torchscript"),Dr=a(" flag."),uo=d(),B=r("h2"),O=r("a"),Ut=r("span"),f(be.$$.fragment),Fr=d(),Jt=r("span"),Yr=a("Dummy inputs and standard lengths"),wo=d(),L=r("p"),qr=a(`The dummy inputs are used for a models forward pass. While the inputs\u2019 values are
propagated through the layers, PyTorch keeps track of the different operations executed
on each tensor. These recorded operations are then used to create the `),jt=r("em"),Kr=a("trace"),Or=a(` of the
model.`),yo=d(),Le=r("p"),Lr=a(`The trace is created relative to the inputs\u2019 dimensions. It is therefore constrained by
the dimensions of the dummy input, and will not work for any other sequence length or
batch size. When trying with a different size, the following error is raised:`),vo=d(),f(Me.$$.fragment),bo=d(),et=r("p"),en=a(`We recommended you trace the model with a dummy input size at least as large as the
largest input that will be fed to the model during inference. Padding can help fill the
missing values. However, since the model is traced with a larger input size, the
dimensions of the matrix will also be large, resulting in more calculations.`),Mo=d(),tt=r("p"),tn=a(`Be careful of the total number of operations done on each input and follow the
performance closely when exporting varying sequence-length models.`),go=d(),C=r("h2"),ee=r("a"),$t=r("span"),f(ge.$$.fragment),on=d(),St=r("span"),rn=a("Using TorchScript in Python"),Eo=d(),ot=r("p"),nn=a(`This section demonstrates how to save and load models as well as how to use the trace
for inference.`),To=d(),Z=r("h3"),te=r("a"),kt=r("span"),f(Ee.$$.fragment),ln=d(),Wt=r("span"),an=a("Saving a model"),_o=d(),E=r("p"),sn=a("To export a "),At=r("code"),hn=a("BertModel"),cn=a(" with TorchScript, instantiate "),It=r("code"),dn=a("BertModel"),pn=a(" from the "),Nt=r("code"),fn=a("BertConfig"),mn=a(`
class and then save it to disk under the filename `),Bt=r("code"),un=a("traced_bert.pt"),wn=a(":"),Uo=d(),f(Te.$$.fragment),Jo=d(),x=r("h3"),oe=r("a"),Ct=r("span"),f(_e.$$.fragment),yn=d(),Zt=r("span"),vn=a("Loading a model"),jo=d(),J=r("p"),bn=a("Now you can load the previously saved "),xt=r("code"),Mn=a("BertModel"),gn=a(", "),zt=r("code"),En=a("traced_bert.pt"),Tn=a(`, from disk and use
it on the previously initialised `),Rt=r("code"),_n=a("dummy_input"),Un=a(":"),$o=d(),f(Ue.$$.fragment),So=d(),z=r("h3"),re=r("a"),Pt=r("span"),f(Je.$$.fragment),Jn=d(),Qt=r("span"),jn=a("Using a traced model for inference"),ko=d(),ne=r("p"),$n=a("Use the traced model for inference by using its "),Vt=r("code"),Sn=a("__call__"),kn=a(" dunder method:"),Wo=d(),f(je.$$.fragment),Ao=d(),R=r("h2"),le=r("a"),Gt=r("span"),f($e.$$.fragment),Wn=d(),Ht=r("span"),An=a("Deploy Hugging Face TorchScript models to AWS with the Neuron SDK"),Io=d(),k=r("p"),In=a("AWS introduced the "),Se=r("a"),Nn=a("Amazon EC2 Inf1"),Bn=a(`
instance family for low cost, high performance machine learning inference in the cloud.
The Inf1 instances are powered by the AWS Inferentia chip, a custom-built hardware
accelerator, specializing in deep learning inferencing workloads. `),ke=r("a"),Cn=a(`AWS
Neuron`),Zn=a(` is the SDK for
Inferentia that supports tracing and optimizing transformers models for deployment on
Inf1. The Neuron SDK provides:`),No=d(),W=r("ol"),Xt=r("li"),xn=a(`Easy-to-use API with one line of code change to trace and optimize a TorchScript
model for inference in the cloud.`),zn=d(),We=r("li"),Rn=a("Out of the box performance optimizations for "),Ae=r("a"),Pn=a(`improved
cost-performance`),Qn=a("."),Vn=d(),P=r("li"),Gn=a(`Support for Hugging Face transformers models built with either
`),Ie=r("a"),Hn=a("PyTorch"),Xn=a(`
or
`),Ne=r("a"),Dn=a("TensorFlow"),Fn=a("."),Bo=d(),Q=r("h3"),ae=r("a"),Dt=r("span"),f(Be.$$.fragment),Yn=d(),Ft=r("span"),qn=a("Implications"),Co=d(),M=r("p"),Kn=a("Transformers models based on the "),Ce=r("a"),On=a(`BERT (Bidirectional Encoder Representations from
Transformers)`),Ln=a(`
architecture, or its variants such as
`),Ze=r("a"),el=a("distilBERT"),tl=a(` and
`),xe=r("a"),ol=a("roBERTa"),rl=a(` run best on
Inf1 for non-generative tasks such as extractive question answering, sequence
classification, and token classification. However, text generation tasks can still be
adapted to run on Inf1 according to this `),ze=r("a"),nl=a(`AWS Neuron MarianMT
tutorial`),ll=a(`.
More information about models that can be converted out of the box on Inferentia can be
found in the `),Re=r("a"),al=a(`Model Architecture
Fit`),sl=a(`
section of the Neuron documentation.`),Zo=d(),V=r("h3"),se=r("a"),Yt=r("span"),f(Pe.$$.fragment),il=d(),qt=r("span"),hl=a("Dependencies"),xo=d(),A=r("p"),cl=a("Using AWS Neuron to convert models requires a "),Qe=r("a"),dl=a(`Neuron SDK
environment`),pl=a(`
which comes preconfigured on `),Ve=r("a"),fl=a(`AWS Deep Learning
AMI`),ml=a("."),zo=d(),G=r("h3"),ie=r("a"),Kt=r("span"),f(Ge.$$.fragment),ul=d(),Ot=r("span"),wl=a("Converting a model for AWS Neuron"),Ro=d(),j=r("p"),yl=a("Convert a model for AWS NEURON using the same code from "),rt=r("a"),vl=a(`Using TorchScript in
Python`),bl=a(" to trace a "),Lt=r("code"),Ml=a("BertModel"),gl=a(`. Import the
`),eo=r("code"),El=a("torch.neuron"),Tl=a(` framework extension to access the components of the Neuron SDK through a
Python API:`),Po=d(),f(He.$$.fragment),Qo=d(),nt=r("p"),_l=a("You only need to modify the following line:"),Vo=d(),f(Xe.$$.fragment),Go=d(),lt=r("p"),Ul=a("This enables the Neuron SDK to trace the model and optimize it for Inf1 instances."),Ho=d(),he=r("p"),Jl=a(`To learn more about AWS Neuron SDK features, tools, example tutorials and latest
updates, please see the `),De=r("a"),jl=a(`AWS NeuronSDK
documentation`),$l=a("."),this.h()},l(e){const i=ls('[data-svelte="svelte-1phssyn"]',document.head);b=n(i,"META",{name:!0,content:!0}),i.forEach(o),H=p(e),g=n(e,"H1",{class:!0});var Fe=l(g);T=n(Fe,"A",{id:!0,class:!0,href:!0});var Sl=l(T);dt=n(Sl,"SPAN",{});var kl=l(dt);m(me.$$.fragment,kl),kl.forEach(o),Sl.forEach(o),pr=p(Fe),pt=n(Fe,"SPAN",{});var Wl=l(pt);fr=s(Wl,"Export to TorchScript"),Wl.forEach(o),Fe.forEach(o),oo=p(e),m(X.$$.fragment,e),ro=p(e),D=n(e,"P",{});var Do=l(D);mr=s(Do,"According to the "),ue=n(Do,"A",{href:!0,rel:!0});var Al=l(ue);ur=s(Al,"TorchScript documentation"),Al.forEach(o),wr=s(Do,":"),Do.forEach(o),no=p(e),Ye=n(e,"BLOCKQUOTE",{});var Il=l(Ye);ft=n(Il,"P",{});var Nl=l(ft);yr=s(Nl,"TorchScript is a way to create serializable and optimizable models from PyTorch code."),Nl.forEach(o),Il.forEach(o),lo=p(e),F=n(e,"P",{});var Fo=l(F);vr=s(Fo,"There are two PyTorch modules, "),we=n(Fo,"A",{href:!0,rel:!0});var Bl=l(we);br=s(Bl,`JIT and
TRACE`),Bl.forEach(o),Mr=s(Fo,`, that allow developers to export their
models to be reused in other programs like efficiency-oriented C++ programs.`),Fo.forEach(o),ao=p(e),qe=n(e,"P",{});var Cl=l(qe);gr=s(Cl,`We provide an interface that allows you to export \u{1F917} Transformers models to TorchScript
so they can be reused in a different environment than PyTorch-based Python programs.
Here, we explain how to export and use our models using TorchScript.`),Cl.forEach(o),so=p(e),Ke=n(e,"P",{});var Zl=l(Ke);Er=s(Zl,"Exporting a model requires two things:"),Zl.forEach(o),io=p(e),Y=n(e,"UL",{});var Yo=l(Y);ye=n(Yo,"LI",{});var qo=l(ye);Tr=s(qo,"model instantiation with the "),mt=n(qo,"CODE",{});var xl=l(mt);_r=s(xl,"torchscript"),xl.forEach(o),Ur=s(qo," flag"),qo.forEach(o),Jr=p(Yo),ut=n(Yo,"LI",{});var zl=l(ut);jr=s(zl,"a forward pass with dummy inputs"),zl.forEach(o),Yo.forEach(o),ho=p(e),Oe=n(e,"P",{});var Rl=l(Oe);$r=s(Rl,`These necessities imply several things developers should be careful about as detailed
below.`),Rl.forEach(o),co=p(e),N=n(e,"H2",{class:!0});var Ko=l(N);q=n(Ko,"A",{id:!0,class:!0,href:!0});var Pl=l(q);wt=n(Pl,"SPAN",{});var Ql=l(wt);m(ve.$$.fragment,Ql),Ql.forEach(o),Pl.forEach(o),Sr=p(Ko),yt=n(Ko,"SPAN",{});var Vl=l(yt);kr=s(Vl,"TorchScript flag and tied weights"),Vl.forEach(o),Ko.forEach(o),po=p(e),_=n(e,"P",{});var ce=l(_);Wr=s(ce,"The "),vt=n(ce,"CODE",{});var Gl=l(vt);Ar=s(Gl,"torchscript"),Gl.forEach(o),Ir=s(ce,` flag is necessary because most of the \u{1F917} Transformers language models
have tied weights between their `),bt=n(ce,"CODE",{});var Hl=l(bt);Nr=s(Hl,"Embedding"),Hl.forEach(o),Br=s(ce," layer and their "),Mt=n(ce,"CODE",{});var Xl=l(Mt);Cr=s(Xl,"Decoding"),Xl.forEach(o),Zr=s(ce,` layer.
TorchScript does not allow you to export models that have tied weights, so it is
necessary to untie and clone the weights beforehand.`),ce.forEach(o),fo=p(e),U=n(e,"P",{});var de=l(U);xr=s(de,"Models instantiated with the "),gt=n(de,"CODE",{});var Dl=l(gt);zr=s(Dl,"torchscript"),Dl.forEach(o),Rr=s(de," flag have their "),Et=n(de,"CODE",{});var Fl=l(Et);Pr=s(Fl,"Embedding"),Fl.forEach(o),Qr=s(de,` layer and
`),Tt=n(de,"CODE",{});var Yl=l(Tt);Vr=s(Yl,"Decoding"),Yl.forEach(o),Gr=s(de,` layer separated, which means that they should not be trained down the line.
Training would desynchronize the two layers, leading to unexpected results.`),de.forEach(o),mo=p(e),K=n(e,"P",{});var Oo=l(K);Hr=s(Oo,`This is not the case for models that do not have a language model head, as those do not
have tied weights. These models can be safely exported without the `),_t=n(Oo,"CODE",{});var ql=l(_t);Xr=s(ql,"torchscript"),ql.forEach(o),Dr=s(Oo," flag."),Oo.forEach(o),uo=p(e),B=n(e,"H2",{class:!0});var Lo=l(B);O=n(Lo,"A",{id:!0,class:!0,href:!0});var Kl=l(O);Ut=n(Kl,"SPAN",{});var Ol=l(Ut);m(be.$$.fragment,Ol),Ol.forEach(o),Kl.forEach(o),Fr=p(Lo),Jt=n(Lo,"SPAN",{});var Ll=l(Jt);Yr=s(Ll,"Dummy inputs and standard lengths"),Ll.forEach(o),Lo.forEach(o),wo=p(e),L=n(e,"P",{});var er=l(L);qr=s(er,`The dummy inputs are used for a models forward pass. While the inputs\u2019 values are
propagated through the layers, PyTorch keeps track of the different operations executed
on each tensor. These recorded operations are then used to create the `),jt=n(er,"EM",{});var ea=l(jt);Kr=s(ea,"trace"),ea.forEach(o),Or=s(er,` of the
model.`),er.forEach(o),yo=p(e),Le=n(e,"P",{});var ta=l(Le);Lr=s(ta,`The trace is created relative to the inputs\u2019 dimensions. It is therefore constrained by
the dimensions of the dummy input, and will not work for any other sequence length or
batch size. When trying with a different size, the following error is raised:`),ta.forEach(o),vo=p(e),m(Me.$$.fragment,e),bo=p(e),et=n(e,"P",{});var oa=l(et);en=s(oa,`We recommended you trace the model with a dummy input size at least as large as the
largest input that will be fed to the model during inference. Padding can help fill the
missing values. However, since the model is traced with a larger input size, the
dimensions of the matrix will also be large, resulting in more calculations.`),oa.forEach(o),Mo=p(e),tt=n(e,"P",{});var ra=l(tt);tn=s(ra,`Be careful of the total number of operations done on each input and follow the
performance closely when exporting varying sequence-length models.`),ra.forEach(o),go=p(e),C=n(e,"H2",{class:!0});var tr=l(C);ee=n(tr,"A",{id:!0,class:!0,href:!0});var na=l(ee);$t=n(na,"SPAN",{});var la=l($t);m(ge.$$.fragment,la),la.forEach(o),na.forEach(o),on=p(tr),St=n(tr,"SPAN",{});var aa=l(St);rn=s(aa,"Using TorchScript in Python"),aa.forEach(o),tr.forEach(o),Eo=p(e),ot=n(e,"P",{});var sa=l(ot);nn=s(sa,`This section demonstrates how to save and load models as well as how to use the trace
for inference.`),sa.forEach(o),To=p(e),Z=n(e,"H3",{class:!0});var or=l(Z);te=n(or,"A",{id:!0,class:!0,href:!0});var ia=l(te);kt=n(ia,"SPAN",{});var ha=l(kt);m(Ee.$$.fragment,ha),ha.forEach(o),ia.forEach(o),ln=p(or),Wt=n(or,"SPAN",{});var ca=l(Wt);an=s(ca,"Saving a model"),ca.forEach(o),or.forEach(o),_o=p(e),E=n(e,"P",{});var I=l(E);sn=s(I,"To export a "),At=n(I,"CODE",{});var da=l(At);hn=s(da,"BertModel"),da.forEach(o),cn=s(I," with TorchScript, instantiate "),It=n(I,"CODE",{});var pa=l(It);dn=s(pa,"BertModel"),pa.forEach(o),pn=s(I," from the "),Nt=n(I,"CODE",{});var fa=l(Nt);fn=s(fa,"BertConfig"),fa.forEach(o),mn=s(I,`
class and then save it to disk under the filename `),Bt=n(I,"CODE",{});var ma=l(Bt);un=s(ma,"traced_bert.pt"),ma.forEach(o),wn=s(I,":"),I.forEach(o),Uo=p(e),m(Te.$$.fragment,e),Jo=p(e),x=n(e,"H3",{class:!0});var rr=l(x);oe=n(rr,"A",{id:!0,class:!0,href:!0});var ua=l(oe);Ct=n(ua,"SPAN",{});var wa=l(Ct);m(_e.$$.fragment,wa),wa.forEach(o),ua.forEach(o),yn=p(rr),Zt=n(rr,"SPAN",{});var ya=l(Zt);vn=s(ya,"Loading a model"),ya.forEach(o),rr.forEach(o),jo=p(e),J=n(e,"P",{});var pe=l(J);bn=s(pe,"Now you can load the previously saved "),xt=n(pe,"CODE",{});var va=l(xt);Mn=s(va,"BertModel"),va.forEach(o),gn=s(pe,", "),zt=n(pe,"CODE",{});var ba=l(zt);En=s(ba,"traced_bert.pt"),ba.forEach(o),Tn=s(pe,`, from disk and use
it on the previously initialised `),Rt=n(pe,"CODE",{});var Ma=l(Rt);_n=s(Ma,"dummy_input"),Ma.forEach(o),Un=s(pe,":"),pe.forEach(o),$o=p(e),m(Ue.$$.fragment,e),So=p(e),z=n(e,"H3",{class:!0});var nr=l(z);re=n(nr,"A",{id:!0,class:!0,href:!0});var ga=l(re);Pt=n(ga,"SPAN",{});var Ea=l(Pt);m(Je.$$.fragment,Ea),Ea.forEach(o),ga.forEach(o),Jn=p(nr),Qt=n(nr,"SPAN",{});var Ta=l(Qt);jn=s(Ta,"Using a traced model for inference"),Ta.forEach(o),nr.forEach(o),ko=p(e),ne=n(e,"P",{});var lr=l(ne);$n=s(lr,"Use the traced model for inference by using its "),Vt=n(lr,"CODE",{});var _a=l(Vt);Sn=s(_a,"__call__"),_a.forEach(o),kn=s(lr," dunder method:"),lr.forEach(o),Wo=p(e),m(je.$$.fragment,e),Ao=p(e),R=n(e,"H2",{class:!0});var ar=l(R);le=n(ar,"A",{id:!0,class:!0,href:!0});var Ua=l(le);Gt=n(Ua,"SPAN",{});var Ja=l(Gt);m($e.$$.fragment,Ja),Ja.forEach(o),Ua.forEach(o),Wn=p(ar),Ht=n(ar,"SPAN",{});var ja=l(Ht);An=s(ja,"Deploy Hugging Face TorchScript models to AWS with the Neuron SDK"),ja.forEach(o),ar.forEach(o),Io=p(e),k=n(e,"P",{});var at=l(k);In=s(at,"AWS introduced the "),Se=n(at,"A",{href:!0,rel:!0});var $a=l(Se);Nn=s($a,"Amazon EC2 Inf1"),$a.forEach(o),Bn=s(at,`
instance family for low cost, high performance machine learning inference in the cloud.
The Inf1 instances are powered by the AWS Inferentia chip, a custom-built hardware
accelerator, specializing in deep learning inferencing workloads. `),ke=n(at,"A",{href:!0,rel:!0});var Sa=l(ke);Cn=s(Sa,`AWS
Neuron`),Sa.forEach(o),Zn=s(at,` is the SDK for
Inferentia that supports tracing and optimizing transformers models for deployment on
Inf1. The Neuron SDK provides:`),at.forEach(o),No=p(e),W=n(e,"OL",{});var st=l(W);Xt=n(st,"LI",{});var ka=l(Xt);xn=s(ka,`Easy-to-use API with one line of code change to trace and optimize a TorchScript
model for inference in the cloud.`),ka.forEach(o),zn=p(st),We=n(st,"LI",{});var sr=l(We);Rn=s(sr,"Out of the box performance optimizations for "),Ae=n(sr,"A",{href:!0,rel:!0});var Wa=l(Ae);Pn=s(Wa,`improved
cost-performance`),Wa.forEach(o),Qn=s(sr,"."),sr.forEach(o),Vn=p(st),P=n(st,"LI",{});var it=l(P);Gn=s(it,`Support for Hugging Face transformers models built with either
`),Ie=n(it,"A",{href:!0,rel:!0});var Aa=l(Ie);Hn=s(Aa,"PyTorch"),Aa.forEach(o),Xn=s(it,`
or
`),Ne=n(it,"A",{href:!0,rel:!0});var Ia=l(Ne);Dn=s(Ia,"TensorFlow"),Ia.forEach(o),Fn=s(it,"."),it.forEach(o),st.forEach(o),Bo=p(e),Q=n(e,"H3",{class:!0});var ir=l(Q);ae=n(ir,"A",{id:!0,class:!0,href:!0});var Na=l(ae);Dt=n(Na,"SPAN",{});var Ba=l(Dt);m(Be.$$.fragment,Ba),Ba.forEach(o),Na.forEach(o),Yn=p(ir),Ft=n(ir,"SPAN",{});var Ca=l(Ft);qn=s(Ca,"Implications"),Ca.forEach(o),ir.forEach(o),Co=p(e),M=n(e,"P",{});var $=l(M);Kn=s($,"Transformers models based on the "),Ce=n($,"A",{href:!0,rel:!0});var Za=l(Ce);On=s(Za,`BERT (Bidirectional Encoder Representations from
Transformers)`),Za.forEach(o),Ln=s($,`
architecture, or its variants such as
`),Ze=n($,"A",{href:!0,rel:!0});var xa=l(Ze);el=s(xa,"distilBERT"),xa.forEach(o),tl=s($,` and
`),xe=n($,"A",{href:!0,rel:!0});var za=l(xe);ol=s(za,"roBERTa"),za.forEach(o),rl=s($,` run best on
Inf1 for non-generative tasks such as extractive question answering, sequence
classification, and token classification. However, text generation tasks can still be
adapted to run on Inf1 according to this `),ze=n($,"A",{href:!0,rel:!0});var Ra=l(ze);nl=s(Ra,`AWS Neuron MarianMT
tutorial`),Ra.forEach(o),ll=s($,`.
More information about models that can be converted out of the box on Inferentia can be
found in the `),Re=n($,"A",{href:!0,rel:!0});var Pa=l(Re);al=s(Pa,`Model Architecture
Fit`),Pa.forEach(o),sl=s($,`
section of the Neuron documentation.`),$.forEach(o),Zo=p(e),V=n(e,"H3",{class:!0});var hr=l(V);se=n(hr,"A",{id:!0,class:!0,href:!0});var Qa=l(se);Yt=n(Qa,"SPAN",{});var Va=l(Yt);m(Pe.$$.fragment,Va),Va.forEach(o),Qa.forEach(o),il=p(hr),qt=n(hr,"SPAN",{});var Ga=l(qt);hl=s(Ga,"Dependencies"),Ga.forEach(o),hr.forEach(o),xo=p(e),A=n(e,"P",{});var ht=l(A);cl=s(ht,"Using AWS Neuron to convert models requires a "),Qe=n(ht,"A",{href:!0,rel:!0});var Ha=l(Qe);dl=s(Ha,`Neuron SDK
environment`),Ha.forEach(o),pl=s(ht,`
which comes preconfigured on `),Ve=n(ht,"A",{href:!0,rel:!0});var Xa=l(Ve);fl=s(Xa,`AWS Deep Learning
AMI`),Xa.forEach(o),ml=s(ht,"."),ht.forEach(o),zo=p(e),G=n(e,"H3",{class:!0});var cr=l(G);ie=n(cr,"A",{id:!0,class:!0,href:!0});var Da=l(ie);Kt=n(Da,"SPAN",{});var Fa=l(Kt);m(Ge.$$.fragment,Fa),Fa.forEach(o),Da.forEach(o),ul=p(cr),Ot=n(cr,"SPAN",{});var Ya=l(Ot);wl=s(Ya,"Converting a model for AWS Neuron"),Ya.forEach(o),cr.forEach(o),Ro=p(e),j=n(e,"P",{});var fe=l(j);yl=s(fe,"Convert a model for AWS NEURON using the same code from "),rt=n(fe,"A",{href:!0});var qa=l(rt);vl=s(qa,`Using TorchScript in
Python`),qa.forEach(o),bl=s(fe," to trace a "),Lt=n(fe,"CODE",{});var Ka=l(Lt);Ml=s(Ka,"BertModel"),Ka.forEach(o),gl=s(fe,`. Import the
`),eo=n(fe,"CODE",{});var Oa=l(eo);El=s(Oa,"torch.neuron"),Oa.forEach(o),Tl=s(fe,` framework extension to access the components of the Neuron SDK through a
Python API:`),fe.forEach(o),Po=p(e),m(He.$$.fragment,e),Qo=p(e),nt=n(e,"P",{});var La=l(nt);_l=s(La,"You only need to modify the following line:"),La.forEach(o),Vo=p(e),m(Xe.$$.fragment,e),Go=p(e),lt=n(e,"P",{});var es=l(lt);Ul=s(es,"This enables the Neuron SDK to trace the model and optimize it for Inf1 instances."),es.forEach(o),Ho=p(e),he=n(e,"P",{});var dr=l(he);Jl=s(dr,`To learn more about AWS Neuron SDK features, tools, example tutorials and latest
updates, please see the `),De=n(dr,"A",{href:!0,rel:!0});var ts=l(De);jl=s(ts,`AWS NeuronSDK
documentation`),ts.forEach(o),$l=s(dr,"."),dr.forEach(o),this.h()},h(){c(b,"name","hf:doc:metadata"),c(b,"content",JSON.stringify(cs)),c(T,"id","export-to-torchscript"),c(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(T,"href","#export-to-torchscript"),c(g,"class","relative group"),c(ue,"href","https://pytorch.org/docs/stable/jit.html"),c(ue,"rel","nofollow"),c(we,"href","https://pytorch.org/docs/stable/jit.html"),c(we,"rel","nofollow"),c(q,"id","torchscript-flag-and-tied-weights"),c(q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(q,"href","#torchscript-flag-and-tied-weights"),c(N,"class","relative group"),c(O,"id","dummy-inputs-and-standard-lengths"),c(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(O,"href","#dummy-inputs-and-standard-lengths"),c(B,"class","relative group"),c(ee,"id","using-torchscript-in-python"),c(ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ee,"href","#using-torchscript-in-python"),c(C,"class","relative group"),c(te,"id","saving-a-model"),c(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(te,"href","#saving-a-model"),c(Z,"class","relative group"),c(oe,"id","loading-a-model"),c(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(oe,"href","#loading-a-model"),c(x,"class","relative group"),c(re,"id","using-a-traced-model-for-inference"),c(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(re,"href","#using-a-traced-model-for-inference"),c(z,"class","relative group"),c(le,"id","deploy-hugging-face-torchscript-models-to-aws-with-the-neuron-sdk"),c(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(le,"href","#deploy-hugging-face-torchscript-models-to-aws-with-the-neuron-sdk"),c(R,"class","relative group"),c(Se,"href","https://aws.amazon.com/ec2/instance-types/inf1/"),c(Se,"rel","nofollow"),c(ke,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/#"),c(ke,"rel","nofollow"),c(Ae,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/benchmark/%3E"),c(Ae,"rel","nofollow"),c(Ie,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/src/examples/pytorch/bert_tutorial/tutorial_pretrained_bert.html"),c(Ie,"rel","nofollow"),c(Ne,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/src/examples/tensorflow/huggingface_bert/huggingface_bert.html"),c(Ne,"rel","nofollow"),c(ae,"id","implications"),c(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ae,"href","#implications"),c(Q,"class","relative group"),c(Ce,"href","https://huggingface.co/docs/transformers/main/model_doc/bert"),c(Ce,"rel","nofollow"),c(Ze,"href","https://huggingface.co/docs/transformers/main/model_doc/distilbert"),c(Ze,"rel","nofollow"),c(xe,"href","https://huggingface.co/docs/transformers/main/model_doc/roberta"),c(xe,"rel","nofollow"),c(ze,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/src/examples/pytorch/transformers-marianmt.html"),c(ze,"rel","nofollow"),c(Re,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/models/models-inferentia.html#models-inferentia"),c(Re,"rel","nofollow"),c(se,"id","dependencies"),c(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(se,"href","#dependencies"),c(V,"class","relative group"),c(Qe,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-frameworks/pytorch-neuron/index.html#installation-guide"),c(Qe,"rel","nofollow"),c(Ve,"href","https://docs.aws.amazon.com/dlami/latest/devguide/tutorial-inferentia-launching.html"),c(Ve,"rel","nofollow"),c(ie,"id","converting-a-model-for-aws-neuron"),c(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ie,"href","#converting-a-model-for-aws-neuron"),c(G,"class","relative group"),c(rt,"href","torchscript#using-torchscript-in-python"),c(De,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/index.html"),c(De,"rel","nofollow")},m(e,i){t(document.head,b),h(e,H,i),h(e,g,i),t(g,T),t(T,dt),u(me,dt,null),t(g,pr),t(g,pt),t(pt,fr),h(e,oo,i),u(X,e,i),h(e,ro,i),h(e,D,i),t(D,mr),t(D,ue),t(ue,ur),t(D,wr),h(e,no,i),h(e,Ye,i),t(Ye,ft),t(ft,yr),h(e,lo,i),h(e,F,i),t(F,vr),t(F,we),t(we,br),t(F,Mr),h(e,ao,i),h(e,qe,i),t(qe,gr),h(e,so,i),h(e,Ke,i),t(Ke,Er),h(e,io,i),h(e,Y,i),t(Y,ye),t(ye,Tr),t(ye,mt),t(mt,_r),t(ye,Ur),t(Y,Jr),t(Y,ut),t(ut,jr),h(e,ho,i),h(e,Oe,i),t(Oe,$r),h(e,co,i),h(e,N,i),t(N,q),t(q,wt),u(ve,wt,null),t(N,Sr),t(N,yt),t(yt,kr),h(e,po,i),h(e,_,i),t(_,Wr),t(_,vt),t(vt,Ar),t(_,Ir),t(_,bt),t(bt,Nr),t(_,Br),t(_,Mt),t(Mt,Cr),t(_,Zr),h(e,fo,i),h(e,U,i),t(U,xr),t(U,gt),t(gt,zr),t(U,Rr),t(U,Et),t(Et,Pr),t(U,Qr),t(U,Tt),t(Tt,Vr),t(U,Gr),h(e,mo,i),h(e,K,i),t(K,Hr),t(K,_t),t(_t,Xr),t(K,Dr),h(e,uo,i),h(e,B,i),t(B,O),t(O,Ut),u(be,Ut,null),t(B,Fr),t(B,Jt),t(Jt,Yr),h(e,wo,i),h(e,L,i),t(L,qr),t(L,jt),t(jt,Kr),t(L,Or),h(e,yo,i),h(e,Le,i),t(Le,Lr),h(e,vo,i),u(Me,e,i),h(e,bo,i),h(e,et,i),t(et,en),h(e,Mo,i),h(e,tt,i),t(tt,tn),h(e,go,i),h(e,C,i),t(C,ee),t(ee,$t),u(ge,$t,null),t(C,on),t(C,St),t(St,rn),h(e,Eo,i),h(e,ot,i),t(ot,nn),h(e,To,i),h(e,Z,i),t(Z,te),t(te,kt),u(Ee,kt,null),t(Z,ln),t(Z,Wt),t(Wt,an),h(e,_o,i),h(e,E,i),t(E,sn),t(E,At),t(At,hn),t(E,cn),t(E,It),t(It,dn),t(E,pn),t(E,Nt),t(Nt,fn),t(E,mn),t(E,Bt),t(Bt,un),t(E,wn),h(e,Uo,i),u(Te,e,i),h(e,Jo,i),h(e,x,i),t(x,oe),t(oe,Ct),u(_e,Ct,null),t(x,yn),t(x,Zt),t(Zt,vn),h(e,jo,i),h(e,J,i),t(J,bn),t(J,xt),t(xt,Mn),t(J,gn),t(J,zt),t(zt,En),t(J,Tn),t(J,Rt),t(Rt,_n),t(J,Un),h(e,$o,i),u(Ue,e,i),h(e,So,i),h(e,z,i),t(z,re),t(re,Pt),u(Je,Pt,null),t(z,Jn),t(z,Qt),t(Qt,jn),h(e,ko,i),h(e,ne,i),t(ne,$n),t(ne,Vt),t(Vt,Sn),t(ne,kn),h(e,Wo,i),u(je,e,i),h(e,Ao,i),h(e,R,i),t(R,le),t(le,Gt),u($e,Gt,null),t(R,Wn),t(R,Ht),t(Ht,An),h(e,Io,i),h(e,k,i),t(k,In),t(k,Se),t(Se,Nn),t(k,Bn),t(k,ke),t(ke,Cn),t(k,Zn),h(e,No,i),h(e,W,i),t(W,Xt),t(Xt,xn),t(W,zn),t(W,We),t(We,Rn),t(We,Ae),t(Ae,Pn),t(We,Qn),t(W,Vn),t(W,P),t(P,Gn),t(P,Ie),t(Ie,Hn),t(P,Xn),t(P,Ne),t(Ne,Dn),t(P,Fn),h(e,Bo,i),h(e,Q,i),t(Q,ae),t(ae,Dt),u(Be,Dt,null),t(Q,Yn),t(Q,Ft),t(Ft,qn),h(e,Co,i),h(e,M,i),t(M,Kn),t(M,Ce),t(Ce,On),t(M,Ln),t(M,Ze),t(Ze,el),t(M,tl),t(M,xe),t(xe,ol),t(M,rl),t(M,ze),t(ze,nl),t(M,ll),t(M,Re),t(Re,al),t(M,sl),h(e,Zo,i),h(e,V,i),t(V,se),t(se,Yt),u(Pe,Yt,null),t(V,il),t(V,qt),t(qt,hl),h(e,xo,i),h(e,A,i),t(A,cl),t(A,Qe),t(Qe,dl),t(A,pl),t(A,Ve),t(Ve,fl),t(A,ml),h(e,zo,i),h(e,G,i),t(G,ie),t(ie,Kt),u(Ge,Kt,null),t(G,ul),t(G,Ot),t(Ot,wl),h(e,Ro,i),h(e,j,i),t(j,yl),t(j,rt),t(rt,vl),t(j,bl),t(j,Lt),t(Lt,Ml),t(j,gl),t(j,eo),t(eo,El),t(j,Tl),h(e,Po,i),u(He,e,i),h(e,Qo,i),h(e,nt,i),t(nt,_l),h(e,Vo,i),u(Xe,e,i),h(e,Go,i),h(e,lt,i),t(lt,Ul),h(e,Ho,i),h(e,he,i),t(he,Jl),t(he,De),t(De,jl),t(he,$l),Xo=!0},p(e,[i]){const Fe={};i&2&&(Fe.$$scope={dirty:i,ctx:e}),X.$set(Fe)},i(e){Xo||(w(me.$$.fragment,e),w(X.$$.fragment,e),w(ve.$$.fragment,e),w(be.$$.fragment,e),w(Me.$$.fragment,e),w(ge.$$.fragment,e),w(Ee.$$.fragment,e),w(Te.$$.fragment,e),w(_e.$$.fragment,e),w(Ue.$$.fragment,e),w(Je.$$.fragment,e),w(je.$$.fragment,e),w($e.$$.fragment,e),w(Be.$$.fragment,e),w(Pe.$$.fragment,e),w(Ge.$$.fragment,e),w(He.$$.fragment,e),w(Xe.$$.fragment,e),Xo=!0)},o(e){y(me.$$.fragment,e),y(X.$$.fragment,e),y(ve.$$.fragment,e),y(be.$$.fragment,e),y(Me.$$.fragment,e),y(ge.$$.fragment,e),y(Ee.$$.fragment,e),y(Te.$$.fragment,e),y(_e.$$.fragment,e),y(Ue.$$.fragment,e),y(Je.$$.fragment,e),y(je.$$.fragment,e),y($e.$$.fragment,e),y(Be.$$.fragment,e),y(Pe.$$.fragment,e),y(Ge.$$.fragment,e),y(He.$$.fragment,e),y(Xe.$$.fragment,e),Xo=!1},d(e){o(b),e&&o(H),e&&o(g),v(me),e&&o(oo),v(X,e),e&&o(ro),e&&o(D),e&&o(no),e&&o(Ye),e&&o(lo),e&&o(F),e&&o(ao),e&&o(qe),e&&o(so),e&&o(Ke),e&&o(io),e&&o(Y),e&&o(ho),e&&o(Oe),e&&o(co),e&&o(N),v(ve),e&&o(po),e&&o(_),e&&o(fo),e&&o(U),e&&o(mo),e&&o(K),e&&o(uo),e&&o(B),v(be),e&&o(wo),e&&o(L),e&&o(yo),e&&o(Le),e&&o(vo),v(Me,e),e&&o(bo),e&&o(et),e&&o(Mo),e&&o(tt),e&&o(go),e&&o(C),v(ge),e&&o(Eo),e&&o(ot),e&&o(To),e&&o(Z),v(Ee),e&&o(_o),e&&o(E),e&&o(Uo),v(Te,e),e&&o(Jo),e&&o(x),v(_e),e&&o(jo),e&&o(J),e&&o($o),v(Ue,e),e&&o(So),e&&o(z),v(Je),e&&o(ko),e&&o(ne),e&&o(Wo),v(je,e),e&&o(Ao),e&&o(R),v($e),e&&o(Io),e&&o(k),e&&o(No),e&&o(W),e&&o(Bo),e&&o(Q),v(Be),e&&o(Co),e&&o(M),e&&o(Zo),e&&o(V),v(Pe),e&&o(xo),e&&o(A),e&&o(zo),e&&o(G),v(Ge),e&&o(Ro),e&&o(j),e&&o(Po),v(He,e),e&&o(Qo),e&&o(nt),e&&o(Vo),v(Xe,e),e&&o(Go),e&&o(lt),e&&o(Ho),e&&o(he)}}}const cs={local:"export-to-torchscript",sections:[{local:"torchscript-flag-and-tied-weights",title:"TorchScript flag and tied weights"},{local:"dummy-inputs-and-standard-lengths",title:"Dummy inputs and standard lengths"},{local:"using-torchscript-in-python",sections:[{local:"saving-a-model",title:"Saving a model"},{local:"loading-a-model",title:"Loading a model"},{local:"using-a-traced-model-for-inference",title:"Using a traced model for inference"}],title:"Using TorchScript in Python"},{local:"deploy-hugging-face-torchscript-models-to-aws-with-the-neuron-sdk",sections:[{local:"implications",title:"Implications"},{local:"dependencies",title:"Dependencies"},{local:"converting-a-model-for-aws-neuron",title:"Converting a model for AWS Neuron"}],title:"Deploy Hugging Face TorchScript models to AWS with the Neuron SDK"}],title:"Export to TorchScript"};function ds(to){return as(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ws extends os{constructor(b){super();rs(this,b,ds,hs,ns,{})}}export{ws as default,cs as metadata};
