import{S as Js,i as ws,s as ks,e as n,k as p,w as y,t as r,M as Ms,c as a,d as o,m as h,a as s,x as v,h as l,b as i,G as e,g,y as J,q as w,o as k,B as M,v as Us,L as yo}from"../../chunks/vendor-hf-doc-builder.js";import{T as vs}from"../../chunks/Tip-hf-doc-builder.js";import{D as He}from"../../chunks/Docstring-hf-doc-builder.js";import{C as qt}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Te}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as bo}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function Ns(j){let c,T;return c=new qt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVE5lb1hKYXBhbmVzZUNvbmZpZyUyQyUyMEdQVE5lb1hKYXBhbmVzZU1vZGVsJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEdQVE5lb1hKYXBhbmVzZSUyMGdwdC1uZW94LWphcGFuZXNlLTIuN2IlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwR1BUTmVvWEphcGFuZXNlQ29uZmlnKCklMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwbW9kZWwlMjAod2l0aCUyMHJhbmRvbSUyMHdlaWdodHMpJTIwZnJvbSUyMHRoZSUyMGdwdC1uZW94LWphcGFuZXNlLTIuN2IlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFtb2RlbCUyMCUzRCUyMEdQVE5lb1hKYXBhbmVzZU1vZGVsKGNvbmZpZ3VyYXRpb24pJTBBJTBBJTIzJTIwQWNjZXNzaW5nJTIwdGhlJTIwbW9kZWwlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMG1vZGVsLmNvbmZpZw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTNeoXJapaneseConfig, GPTNeoXJapaneseModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a GPTNeoXJapanese gpt-neox-japanese-2.7b style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = GPTNeoXJapaneseConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model (with random weights) from the gpt-neox-japanese-2.7b style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = GPTNeoXJapaneseModel(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`}}),{c(){y(c.$$.fragment)},l(u){v(c.$$.fragment,u)},m(u,f){J(c,u,f),T=!0},p:yo,i(u){T||(w(c.$$.fragment,u),T=!0)},o(u){k(c.$$.fragment,u),T=!1},d(u){M(c,u)}}}function $s(j){let c,T,u,f,b;return f=new qt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVE5lb1hKYXBhbmVzZVRva2VuaXplciUwQSUwQXRva2VuaXplciUyMCUzRCUyMEdQVE5lb1hKYXBhbmVzZVRva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQSUyMyUyMFlvdSUyMGNhbiUyMGNvbmZpcm0lMjBib3RoJTIwJUU2JTg1JUI2JUU1JUJGJTlDJTIwYW5kJTIwJUU2JTg1JUI2JUU2JTg3JTg5JTIwYXJlJTIwZW5jb2RlZCUyMHRvJTIwMTc3NDklMEF0b2tlbml6ZXIoJTIyJUU1JTkwJUJFJUU4JUJDJUE5JUUzJTgxJUFGJUU3JThDJUFCJUUzJTgxJUE3JUUzJTgxJTgyJUUzJTgyJThCJUYwJTlGJTkwJUFGJUUzJTgwJTgyJUU1JUFFJTlGJUUzJTgxJUFGJUU2JTg1JUI2JUU1JUJGJTlDKCVFNiU4NSVCNiVFNiU4NyU4OSklRTUlQTQlQTclRTUlQUQlQTYlRTUlODclQkElRTglQkElQUIlMjIpJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTBBJTBBJTIzJTIwQm90aCUyMCVFNiU4NSVCNiVFNSVCRiU5QyUyMGFuZCUyMCVFNiU4NSVCNiVFNiU4NyU4OSUyMGFyZSUyMGRlY29kZWQlMjB0byUyMCVFNiU4NSVCNiVFNSVCRiU5QyUwQXRva2VuaXplci5kZWNvZGUodG9rZW5pemVyKCUyMiVFNSU5MCVCRSVFOCVCQyVBOSVFMyU4MSVBRiVFNyU4QyVBQiVFMyU4MSVBNyVFMyU4MSU4MiVFMyU4MiU4QiVGMCU5RiU5MCVBRiVFMyU4MCU4MiVFNSVBRSU5RiVFMyU4MSVBRiVFNiU4NSVCNiVFNSVCRiU5QyglRTYlODUlQjYlRTYlODclODkpJUU1JUE0JUE3JUU1JUFEJUE2JUU1JTg3JUJBJUU4JUJBJUFCJTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTNeoXJapaneseTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTNeoXJapaneseTokenizer.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># You can confirm both \u6176\u5FDC and \u6176\u61C9 are encoded to 17749</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002\u5B9F\u306F\u6176\u5FDC(\u6176\u61C9)\u5927\u5B66\u51FA\u8EAB&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[<span class="hljs-number">30014</span>, <span class="hljs-number">26883</span>, <span class="hljs-number">26638</span>, <span class="hljs-number">27228</span>, <span class="hljs-number">25</span>, <span class="hljs-number">26650</span>, <span class="hljs-number">31732</span>, <span class="hljs-number">31679</span>, <span class="hljs-number">27809</span>, <span class="hljs-number">26638</span>, <span class="hljs-number">17749</span>, <span class="hljs-number">31592</span>, <span class="hljs-number">17749</span>, <span class="hljs-number">31593</span>, <span class="hljs-number">321</span>, <span class="hljs-number">1281</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Both \u6176\u5FDC and \u6176\u61C9 are decoded to \u6176\u5FDC</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(tokenizer(<span class="hljs-string">&quot;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002\u5B9F\u306F\u6176\u5FDC(\u6176\u61C9)\u5927\u5B66\u51FA\u8EAB&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002\u5B9F\u306F\u6176\u5FDC(\u6176\u5FDC)\u5927\u5B66\u51FA\u8EAB&#x27;</span>`}}),{c(){c=n("p"),T=r("Example:"),u=p(),y(f.$$.fragment)},l(d){c=a(d,"P",{});var _=s(c);T=l(_,"Example:"),_.forEach(o),u=h(d),v(f.$$.fragment,d)},m(d,_){g(d,c,_),e(c,T),g(d,u,_),J(f,d,_),b=!0},p:yo,i(d){b||(w(f.$$.fragment,d),b=!0)},o(d){k(f.$$.fragment,d),b=!1},d(d){d&&o(c),d&&o(u),M(f,d)}}}function Gs(j){let c,T,u,f,b;return{c(){c=n("p"),T=r("Although the recipe for forward pass needs to be defined within this function, one should call the "),u=n("code"),f=r("Module"),b=r(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(d){c=a(d,"P",{});var _=s(c);T=l(_,"Although the recipe for forward pass needs to be defined within this function, one should call the "),u=a(_,"CODE",{});var F=s(u);f=l(F,"Module"),F.forEach(o),b=l(_,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),_.forEach(o)},m(d,_){g(d,c,_),e(c,T),e(c,u),e(u,f),e(c,b)},d(d){d&&o(c)}}}function js(j){let c,T,u,f,b;return f=new qt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBHUFROZW9YSmFwYW5lc2VNb2RlbCUwQWltcG9ydCUyMHRvcmNoJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQW1vZGVsJTIwJTNEJTIwR1BUTmVvWEphcGFuZXNlTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMmFiZWphJTJGZ3B0LW5lb3gtamFwYW5lc2UtMi43YiUyMiklMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIyJUU2JTk3JUE1JUU2JTlDJUFDJUU4JUFBJTlFJUUzJTgxJUFFR1BULW5lb3glRTMlODElOENIdWdnaW5nJTIwRmFjZSVFMyU4MSVBNyVFNCVCRCVCRiVFMyU4MSU4OCVFMyU4MSVCRSVFMyU4MSU5OSVGMCU5RiU5OCU4MCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKSUwQSUwQWxhc3RfaGlkZGVuX3N0YXRlcyUyMCUzRCUyMG91dHB1dHMubGFzdF9oaWRkZW5fc3RhdGU=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, GPTNeoXJapaneseModel
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = GPTNeoXJapaneseModel.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;\u65E5\u672C\u8A9E\u306EGPT-neox\u304CHugging Face\u3067\u4F7F\u3048\u307E\u3059\u{1F600}&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(**inputs)

<span class="hljs-meta">&gt;&gt;&gt; </span>last_hidden_states = outputs.last_hidden_state`}}),{c(){c=n("p"),T=r("Example:"),u=p(),y(f.$$.fragment)},l(d){c=a(d,"P",{});var _=s(c);T=l(_,"Example:"),_.forEach(o),u=h(d),v(f.$$.fragment,d)},m(d,_){g(d,c,_),e(c,T),g(d,u,_),J(f,d,_),b=!0},p:yo,i(d){b||(w(f.$$.fragment,d),b=!0)},o(d){k(f.$$.fragment,d),b=!1},d(d){d&&o(c),d&&o(u),M(f,d)}}}function Es(j){let c,T,u,f,b;return{c(){c=n("p"),T=r("Although the recipe for forward pass needs to be defined within this function, one should call the "),u=n("code"),f=r("Module"),b=r(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(d){c=a(d,"P",{});var _=s(c);T=l(_,"Although the recipe for forward pass needs to be defined within this function, one should call the "),u=a(_,"CODE",{});var F=s(u);f=l(F,"Module"),F.forEach(o),b=l(_,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),_.forEach(o)},m(d,_){g(d,c,_),e(c,T),e(c,u),e(u,f),e(c,b)},d(d){d&&o(c)}}}function Ps(j){let c,T,u,f,b;return f=new qt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBHUFROZW9YSmFwYW5lc2VGb3JDYXVzYWxMTSUyQyUyMEdQVE5lb1hKYXBhbmVzZUNvbmZpZyUwQWltcG9ydCUyMHRvcmNoJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQWNvbmZpZyUyMCUzRCUyMEdQVE5lb1hKYXBhbmVzZUNvbmZpZy5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQWNvbmZpZy5pc19kZWNvZGVyJTIwJTNEJTIwVHJ1ZSUwQW1vZGVsJTIwJTNEJTIwR1BUTmVvWEphcGFuZXNlRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmFiZWphJTJGZ3B0LW5lb3gtamFwYW5lc2UtMi43YiUyMiUyQyUyMGNvbmZpZyUzRGNvbmZpZyklMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIyJUU2JTk3JUE1JUU2JTlDJUFDJUU4JUFBJTlFJUUzJTgxJUFFR1BULW5lb3glRTMlODElOENIdWdnaW5nJTIwRmFjZSVFMyU4MSVBNyVFNCVCRCVCRiVFMyU4MSU4OCVFMyU4MSVCRSVFMyU4MSU5OSVGMCU5RiU5OCU4MCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKSUwQSUwQXByZWRpY3Rpb25fbG9naXRzJTIwJTNEJTIwb3V0cHV0cy5sb2dpdHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseConfig
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>config = GPTNeoXJapaneseConfig.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>config.is_decoder = <span class="hljs-literal">True</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = GPTNeoXJapaneseForCausalLM.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>, config=config)

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;\u65E5\u672C\u8A9E\u306EGPT-neox\u304CHugging Face\u3067\u4F7F\u3048\u307E\u3059\u{1F600}&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(**inputs)

<span class="hljs-meta">&gt;&gt;&gt; </span>prediction_logits = outputs.logits`}}),{c(){c=n("p"),T=r("Example:"),u=p(),y(f.$$.fragment)},l(d){c=a(d,"P",{});var _=s(c);T=l(_,"Example:"),_.forEach(o),u=h(d),v(f.$$.fragment,d)},m(d,_){g(d,c,_),e(c,T),g(d,u,_),J(f,d,_),b=!0},p:yo,i(d){b||(w(f.$$.fragment,d),b=!0)},o(d){k(f.$$.fragment,d),b=!1},d(d){d&&o(c),d&&o(u),M(f,d)}}}function Xs(j){let c,T,u,f,b,d,_,F,vo,At,W,ee,mt,be,Jo,ut,wo,Lt,N,ko,ye,Mo,Uo,ve,No,$o,ft,Go,jo,Je,Eo,Po,we,Xo,Co,Qt,U,zo,ke,xo,Fo,Me,Vo,Zo,Ue,Io,Wo,Ne,Bo,So,$e,Ro,qo,Ge,Ao,Lo,Ot,B,te,gt,je,Qo,_t,Oo,Yt,oe,Yo,Tt,Ho,Do,Ht,Ee,Dt,S,ne,bt,Pe,Ko,yt,en,Kt,De,vt,Ke,tn,eo,R,ae,Jt,Xe,on,wt,nn,to,P,Ce,an,q,sn,kt,rn,ln,ze,dn,cn,pn,A,hn,et,mn,un,tt,fn,gn,_n,se,oo,L,re,Mt,xe,Tn,Ut,bn,no,$,Fe,yn,Q,vn,ot,Jn,wn,Ve,kn,Mn,Un,G,Nt,Nn,$n,$t,Gn,jn,Gt,En,Pn,jt,Xn,Cn,Et,zn,xn,Pt,Fn,Vn,le,Zn,ie,Ze,In,Xt,Wn,ao,O,de,Ct,Ie,Bn,zt,Sn,so,V,We,Rn,Be,qn,Se,An,Ln,Qn,X,Re,On,Y,Yn,nt,Hn,Dn,xt,Kn,ea,ta,ce,oa,pe,ro,H,he,Ft,qe,na,Vt,aa,lo,Z,Ae,sa,D,ra,Zt,la,ia,Le,da,ca,pa,C,Qe,ha,K,ma,at,ua,fa,It,ga,_a,Ta,me,ba,ue,io;return d=new Te({}),be=new Te({}),je=new Te({}),Ee=new qt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVE5lb1hKYXBhbmVzZUZvckNhdXNhbExNJTJDJTIwR1BUTmVvWEphcGFuZXNlVG9rZW5pemVyJTBBJTBBbW9kZWwlMjAlM0QlMjBHUFROZW9YSmFwYW5lc2VGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQXRva2VuaXplciUyMCUzRCUyMEdQVE5lb1hKYXBhbmVzZVRva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMiVFNCVCQSVCQSVFMyU4MSVBOEFJJUUzJTgxJThDJUU1JThEJTk0JUU4JUFBJUJGJUUzJTgxJTk5JUUzJTgyJThCJUUzJTgxJTlGJUUzJTgyJTgxJUUzJTgxJUFCJUUzJTgxJUFGJUUzJTgwJTgxJTIyJTBBJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLmlucHV0X2lkcyUwQSUwQWdlbl90b2tlbnMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSglMEElMjAlMjAlMjAlMjBpbnB1dF9pZHMlMkMlMEElMjAlMjAlMjAlMjBkb19zYW1wbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwdGVtcGVyYXR1cmUlM0QwLjklMkMlMEElMjAlMjAlMjAlMjBtYXhfbGVuZ3RoJTNEMTAwJTJDJTBBKSUwQWdlbl90ZXh0JTIwJTNEJTIwdG9rZW5pemVyLmJhdGNoX2RlY29kZShnZW5fdG9rZW5zJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTVCMCU1RCUwQSUwQXByaW50KGdlbl90ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = GPTNeoXJapaneseForCausalLM.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTNeoXJapaneseTokenizer.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;\u4EBA\u3068AI\u304C\u5354\u8ABF\u3059\u308B\u305F\u3081\u306B\u306F\u3001&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids

<span class="hljs-meta">&gt;&gt;&gt; </span>gen_tokens = model.generate(
<span class="hljs-meta">... </span>    input_ids,
<span class="hljs-meta">... </span>    do_sample=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    temperature=<span class="hljs-number">0.9</span>,
<span class="hljs-meta">... </span>    max_length=<span class="hljs-number">100</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>gen_text = tokenizer.batch_decode(gen_tokens, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(gen_text)
\u4EBA\u3068AI\u304C\u5354\u8ABF\u3059\u308B\u305F\u3081\u306B\u306F\u3001AI\u3068\u4EBA\u304C\u5171\u5B58\u3057\u3001AI\u3092\u6B63\u3057\u304F\u7406\u89E3\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002`}}),Pe=new Te({}),Xe=new Te({}),Ce=new He({props:{name:"class transformers.GPTNeoXJapaneseConfig",anchor:"transformers.GPTNeoXJapaneseConfig",parameters:[{name:"vocab_size",val:" = 32000"},{name:"hidden_size",val:" = 2560"},{name:"num_hidden_layers",val:" = 32"},{name:"num_attention_heads",val:" = 32"},{name:"intermediate_multiple_size",val:" = 4"},{name:"hidden_act",val:" = 'gelu'"},{name:"rotary_pct",val:" = 1.0"},{name:"rotary_emb_base",val:" = 10000"},{name:"max_position_embeddings",val:" = 2048"},{name:"initializer_range",val:" = 0.02"},{name:"layer_norm_eps",val:" = 1e-05"},{name:"use_cache",val:" = True"},{name:"bos_token_id",val:" = 31996"},{name:"eos_token_id",val:" = 31999"},{name:"attention_dropout",val:" = 0.1"},{name:"hidden_dropout",val:" = 0.0"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 32000) &#x2014;
Vocabulary size of the GPTNeoXJapanese model. Defines the number of different tokens that can be
represented by the <code>inputs_ids</code> passed when calling <code>GPTNeoXJapanese</code>.`,name:"vocab_size"},{anchor:"transformers.GPTNeoXJapaneseConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 2560) &#x2014;
Dimension of the encoder layers and the pooler layer.`,name:"hidden_size"},{anchor:"transformers.GPTNeoXJapaneseConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.GPTNeoXJapaneseConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.GPTNeoXJapaneseConfig.intermediate_multiple_size",description:`<strong>intermediate_multiple_size</strong> (<code>int</code>, <em>optional</em>, defaults to 4) &#x2014;
Dimension of the &#x201C;intermediate&#x201D; layer in the Transformer encoder is calculated by hidden_size *
intermediate_multiple_size.`,name:"intermediate_multiple_size"},{anchor:"transformers.GPTNeoXJapaneseConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>function</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the encoder and pooler.`,name:"hidden_act"},{anchor:"transformers.GPTNeoXJapaneseConfig.rotary_pct",description:`<strong>rotary_pct</strong> (<code>float</code>, <em>optional</em>, defaults to 1.00) &#x2014;
percentage of hidden dimensions to allocate to rotary embeddings`,name:"rotary_pct"},{anchor:"transformers.GPTNeoXJapaneseConfig.rotary_emb_base",description:`<strong>rotary_emb_base</strong> (<code>int</code>, <em>optional</em>, defaults to 10000) &#x2014;
base for computing rotary embeddings frequency`,name:"rotary_emb_base"},{anchor:"transformers.GPTNeoXJapaneseConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to 2048) &#x2014;
The maximum sequence length that this model might ever be used with.`,name:"max_position_embeddings"},{anchor:"transformers.GPTNeoXJapaneseConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 0.02) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.GPTNeoXJapaneseConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-5) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.GPTNeoXJapaneseConfig.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if <code>config.is_decoder=True</code>.`,name:"use_cache"},{anchor:"transformers.GPTNeoXJapaneseConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The dropout ratio for the attention.`,name:"attention_dropout"},{anchor:"transformers.GPTNeoXJapaneseConfig.hidden_dropout",description:`<strong>hidden_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout ratio for the hidden layer.
Example &#x2014;`,name:"hidden_dropout"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gpt_neox_japanese/configuration_gpt_neox_japanese.py#L28"}}),se=new bo({props:{anchor:"transformers.GPTNeoXJapaneseConfig.example",$$slots:{default:[Ns]},$$scope:{ctx:j}}}),xe=new Te({}),Fe=new He({props:{name:"class transformers.GPTNeoXJapaneseTokenizer",anchor:"transformers.GPTNeoXJapaneseTokenizer",parameters:[{name:"vocab_file",val:""},{name:"emoji_file",val:""},{name:"unk_token",val:" = '<|endoftext|>'"},{name:"pad_token",val:" = '<|endoftext|>'"},{name:"bos_token",val:" = '<|startoftext|>'"},{name:"eos_token",val:" = '<|endoftext|>'"},{name:"do_clean_text",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
File containing the vocabulary.`,name:"vocab_file"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.emoji_file",description:`<strong>emoji_file</strong> (<code>str</code>) &#x2014;
File containing the emoji.`,name:"emoji_file"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|endoftext|&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|endoftext|&gt;&quot;</code>) &#x2014;
The token used for padding`,name:"pad_token"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|startoftext|&gt;&quot;</code>) &#x2014;
The beginning of sequence token.`,name:"bos_token"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|endoftext|&gt;&quot;</code>) &#x2014;
The end of sequence token.`,name:"eos_token"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.do_clean_text",description:`<strong>do_clean_text</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to clean text for URL, EMAIL, TEL, Japanese DATE and Japanese PRICE.`,name:"do_clean_text"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gpt_neox_japanese/tokenization_gpt_neox_japanese.py#L70"}}),le=new bo({props:{anchor:"transformers.GPTNeoXJapaneseTokenizer.example",$$slots:{default:[$s]},$$scope:{ctx:j}}}),Ze=new He({props:{name:"convert_tokens_to_string",anchor:"transformers.GPTNeoXJapaneseTokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gpt_neox_japanese/tokenization_gpt_neox_japanese.py#L177"}}),Ie=new Te({}),We=new He({props:{name:"class transformers.GPTNeoXJapaneseModel",anchor:"transformers.GPTNeoXJapaneseModel",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.31.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseConfig">~GPTNeoXJapaneseConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/v4.31.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py#L440"}}),Re=new He({props:{name:"forward",anchor:"transformers.GPTNeoXJapaneseModel.forward",parameters:[{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"head_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"past_key_values",val:": typing.Optional[typing.Tuple[typing.Tuple[torch.FloatTensor]]] = None"},{name:"use_cache",val:": typing.Optional[bool] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseModel.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.31.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>.`,name:"input_ids"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p>
<ul>
<li>0 corresponds to a <em>sentence A</em> token,</li>
<li>1 corresponds to a <em>sentence B</em> token.</li>
</ul>`,name:"token_type_ids"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.max_position_embeddings - 1]</code>.`,name:"position_ids"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.head_mask",description:`<strong>head_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(num_heads,)</code> or <code>(num_layers, num_heads)</code>, <em>optional</em>) &#x2014;
Mask to nullify selected heads of the self-attention modules. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"head_mask"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <em>input_ids</em> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.31.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code> of length <code>config.n_layers</code> with each tuple having 4 tensors of shape <code>(batch_size, num_heads, sequence_length - 1, embed_size_per_head)</code>) &#x2014;
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py#L460",returnDescription:`
<p>A <a
  href="/docs/transformers/v4.31.0/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPast"
>transformers.modeling_outputs.BaseModelOutputWithPast</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/v4.31.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseConfig"
>GPTNeoXJapaneseConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) \u2014 Sequence of hidden-states at the output of the last layer of the model.</p>
<p>If <code>past_key_values</code> is used only the last hidden-state of the sequences of shape <code>(batch_size, 1, hidden_size)</code> is output.</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) \u2014 Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and optionally if
<code>config.is_encoder_decoder=True</code> 2 additional tensors of shape <code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
<code>config.is_encoder_decoder=True</code> in the cross-attention blocks) that can be used (see <code>past_key_values</code>
input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/v4.31.0/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPast"
>transformers.modeling_outputs.BaseModelOutputWithPast</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),ce=new vs({props:{$$slots:{default:[Gs]},$$scope:{ctx:j}}}),pe=new bo({props:{anchor:"transformers.GPTNeoXJapaneseModel.forward.example",$$slots:{default:[js]},$$scope:{ctx:j}}}),qe=new Te({}),Ae=new He({props:{name:"class transformers.GPTNeoXJapaneseForCausalLM",anchor:"transformers.GPTNeoXJapaneseForCausalLM",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseForCausalLM.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.31.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseConfig">~GPTNeoXJapaneseConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/v4.31.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py#L594"}}),Qe=new He({props:{name:"forward",anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward",parameters:[{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"head_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"past_key_values",val:": typing.Optional[typing.Tuple[typing.Tuple[torch.FloatTensor]]] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"use_cache",val:": typing.Optional[bool] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.31.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>.`,name:"input_ids"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p>
<ul>
<li>0 corresponds to a <em>sentence A</em> token,</li>
<li>1 corresponds to a <em>sentence B</em> token.</li>
</ul>`,name:"token_type_ids"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.max_position_embeddings - 1]</code>.`,name:"position_ids"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.head_mask",description:`<strong>head_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(num_heads,)</code> or <code>(num_layers, num_heads)</code>, <em>optional</em>) &#x2014;
Mask to nullify selected heads of the self-attention modules. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"head_mask"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <em>input_ids</em> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.31.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>. The two additional tensors are
only required when the model is used as a decoder in a Sequence to Sequence model.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
<code>[-100, 0, ..., config.vocab_size]</code> (see <code>input_ids</code> docstring) Tokens with indices set to <code>-100</code> are
ignored (masked), the loss is only computed for the tokens with labels n <code>[0, ..., config.vocab_size]</code>.`,name:"labels"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py#L613",returnDescription:`
<p>A <a
  href="/docs/transformers/v4.31.0/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithPast"
>transformers.modeling_outputs.CausalLMOutputWithPast</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/v4.31.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseConfig"
>GPTNeoXJapaneseConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) \u2014 Language modeling loss (for next-token prediction).</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) \u2014 Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) \u2014 Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>)</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/v4.31.0/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithPast"
>transformers.modeling_outputs.CausalLMOutputWithPast</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),me=new vs({props:{$$slots:{default:[Es]},$$scope:{ctx:j}}}),ue=new bo({props:{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.example",$$slots:{default:[Ps]},$$scope:{ctx:j}}}),{c(){c=n("meta"),T=p(),u=n("h1"),f=n("a"),b=n("span"),y(d.$$.fragment),_=p(),F=n("span"),vo=r("GPT-NeoX-Japanese"),At=p(),W=n("h2"),ee=n("a"),mt=n("span"),y(be.$$.fragment),Jo=p(),ut=n("span"),wo=r("Overview"),Lt=p(),N=n("p"),ko=r("We introduce GPT-NeoX-Japanese, which is an autoregressive language model for Japanese, trained on top of "),ye=n("a"),Mo=r("https://github.com/EleutherAI/gpt-neox"),Uo=r(`.
Japanese is a unique language with its large vocabulary and a combination of hiragana, katakana, and kanji writing scripts.
To address this distinct structure of the Japanese language, we use a `),ve=n("a"),No=r("special sub-word tokenizer"),$o=r(". We are very grateful to "),ft=n("em"),Go=r("tanreinama"),jo=r(` for open-sourcing this incredibly helpful tokenizer.
Following the recommendations from Google\u2019s research on `),Je=n("a"),Eo=r("PaLM"),Po=r(", we have removed bias parameters from transformer blocks, achieving better model performance. Please refer "),we=n("a"),Xo=r("this article"),Co=r(" in detail."),Qt=p(),U=n("p"),zo=r("Development of the model was led by "),ke=n("a"),xo=r("Shinya Otani"),Fo=r(", "),Me=n("a"),Vo=r("Takayoshi Makabe"),Zo=r(", "),Ue=n("a"),Io=r("Anuj Arora"),Wo=r(", and "),Ne=n("a"),Bo=r("Kyo Hattori"),So=r(" from "),$e=n("a"),Ro=r("ABEJA, Inc."),qo=r(". For more information on this model-building activity, please refer "),Ge=n("a"),Ao=r("here (ja)"),Lo=r("."),Ot=p(),B=n("h3"),te=n("a"),gt=n("span"),y(je.$$.fragment),Qo=p(),_t=n("span"),Oo=r("Generation"),Yt=p(),oe=n("p"),Yo=r("The "),Tt=n("code"),Ho=r("generate()"),Do=r(" method can be used to generate text using GPT NeoX Japanese model."),Ht=p(),y(Ee.$$.fragment),Dt=p(),S=n("h2"),ne=n("a"),bt=n("span"),y(Pe.$$.fragment),Ko=p(),yt=n("span"),en=r("Documentation resources"),Kt=p(),De=n("ul"),vt=n("li"),Ke=n("a"),tn=r("Causal language modeling task guide"),eo=p(),R=n("h2"),ae=n("a"),Jt=n("span"),y(Xe.$$.fragment),on=p(),wt=n("span"),nn=r("GPTNeoXJapaneseConfig"),to=p(),P=n("div"),y(Ce.$$.fragment),an=p(),q=n("p"),sn=r("This is the configuration class to store the configuration of a "),kt=n("code"),rn=r("GPTNeoXModelJapanese"),ln=r(`. It is used to instantiate
a GPTNeoX model according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of the GPTNeoXJapanese
`),ze=n("a"),dn=r("abeja/gpt-neox-japanese-2.7b"),cn=r(" architecture."),pn=p(),A=n("p"),hn=r("Configuration objects inherit from "),et=n("a"),mn=r("PretrainedConfig"),un=r(` and can be used to control the model outputs. Read the
documentation from `),tt=n("a"),fn=r("PretrainedConfig"),gn=r(" for more information. Default configs is set as 2.7B model"),_n=p(),y(se.$$.fragment),oo=p(),L=n("h2"),re=n("a"),Mt=n("span"),y(xe.$$.fragment),Tn=p(),Ut=n("span"),bn=r("GPTNeoXJapaneseTokenizer"),no=p(),$=n("div"),y(Fe.$$.fragment),yn=p(),Q=n("p"),vn=r("This tokenizer inherits from "),ot=n("a"),Jn=r("PreTrainedTokenizer"),wn=r(` and is based on Japanese special Sub-Word-Encoding that is
used in this repository (`),Ve=n("a"),kn=r("https://github.com/tanreinama/Japanese-BPEEncoder_V2"),Mn=r(`). Check the repository for details.
Japanese has a relatively large vocabulary and there is no separation between words. Furthermore, the language is a
combination of hiragana, katakana, and kanji, and variants such as \u201C1\u201D and \u201C\u2460\u201D are often used. In order to cope
with these, this tokenizer has the following features`),Un=p(),G=n("ul"),Nt=n("li"),Nn=r("Subword-by-subword segmentation, which is intermediate between byte strings and morphological analysis."),$n=p(),$t=n("li"),Gn=r(`BPEs are created for each Kanji, Hiragana, and Katakana character, and there are no BPEs that cross character
types, such as Kanji + Hiragana or Hiragana + Katakana.`),jn=p(),Gt=n("li"),En=r("All-byte encoding that does not require <unk>."),Pn=p(),jt=n("li"),Xn=r("Independent of UTF codes such as 2-byte and 3-byte characters"),Cn=p(),Et=n("li"),zn=r("Conversion of heterographs to the same token_id"),xn=p(),Pt=n("li"),Fn=r("Emoji and Emoticon are grouped into 12 types as special tags."),Vn=p(),y(le.$$.fragment),Zn=p(),ie=n("div"),y(Ze.$$.fragment),In=p(),Xt=n("p"),Wn=r("Converts a sequence of tokens (string) in a single string."),ao=p(),O=n("h2"),de=n("a"),Ct=n("span"),y(Ie.$$.fragment),Bn=p(),zt=n("span"),Sn=r("GPTNeoXJapaneseModel"),so=p(),V=n("div"),y(We.$$.fragment),Rn=p(),Be=n("p"),qn=r(`The bare GPTNeoXJapanese Model transformer outputting raw hidden-states without any specific head on top.
This model is a PyTorch `),Se=n("a"),An=r("torch.nn.Module"),Ln=r(` sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),Qn=p(),X=n("div"),y(Re.$$.fragment),On=p(),Y=n("p"),Yn=r("The "),nt=n("a"),Hn=r("GPTNeoXJapaneseModel"),Dn=r(" forward method, overrides the "),xt=n("code"),Kn=r("__call__"),ea=r(" special method."),ta=p(),y(ce.$$.fragment),oa=p(),y(pe.$$.fragment),ro=p(),H=n("h2"),he=n("a"),Ft=n("span"),y(qe.$$.fragment),na=p(),Vt=n("span"),aa=r("GPTNeoXJapaneseForCausalLM"),lo=p(),Z=n("div"),y(Ae.$$.fragment),sa=p(),D=n("p"),ra=r("GPTNeoXJapanese Model with a "),Zt=n("code"),la=r("language modeling"),ia=r(` head on top for Classifier Model fine-tuning.
This model is a PyTorch `),Le=n("a"),da=r("torch.nn.Module"),ca=r(` sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),pa=p(),C=n("div"),y(Qe.$$.fragment),ha=p(),K=n("p"),ma=r("The "),at=n("a"),ua=r("GPTNeoXJapaneseForCausalLM"),fa=r(" forward method, overrides the "),It=n("code"),ga=r("__call__"),_a=r(" special method."),Ta=p(),y(me.$$.fragment),ba=p(),y(ue.$$.fragment),this.h()},l(t){const m=Ms('[data-svelte="svelte-1phssyn"]',document.head);c=a(m,"META",{name:!0,content:!0}),m.forEach(o),T=h(t),u=a(t,"H1",{class:!0});var Oe=s(u);f=a(Oe,"A",{id:!0,class:!0,href:!0});var Wt=s(f);b=a(Wt,"SPAN",{});var Bt=s(b);v(d.$$.fragment,Bt),Bt.forEach(o),Wt.forEach(o),_=h(Oe),F=a(Oe,"SPAN",{});var St=s(F);vo=l(St,"GPT-NeoX-Japanese"),St.forEach(o),Oe.forEach(o),At=h(t),W=a(t,"H2",{class:!0});var Ye=s(W);ee=a(Ye,"A",{id:!0,class:!0,href:!0});var Rt=s(ee);mt=a(Rt,"SPAN",{});var ya=s(mt);v(be.$$.fragment,ya),ya.forEach(o),Rt.forEach(o),Jo=h(Ye),ut=a(Ye,"SPAN",{});var va=s(ut);wo=l(va,"Overview"),va.forEach(o),Ye.forEach(o),Lt=h(t),N=a(t,"P",{});var z=s(N);ko=l(z,"We introduce GPT-NeoX-Japanese, which is an autoregressive language model for Japanese, trained on top of "),ye=a(z,"A",{href:!0,rel:!0});var Ja=s(ye);Mo=l(Ja,"https://github.com/EleutherAI/gpt-neox"),Ja.forEach(o),Uo=l(z,`.
Japanese is a unique language with its large vocabulary and a combination of hiragana, katakana, and kanji writing scripts.
To address this distinct structure of the Japanese language, we use a `),ve=a(z,"A",{href:!0,rel:!0});var wa=s(ve);No=l(wa,"special sub-word tokenizer"),wa.forEach(o),$o=l(z,". We are very grateful to "),ft=a(z,"EM",{});var ka=s(ft);Go=l(ka,"tanreinama"),ka.forEach(o),jo=l(z,` for open-sourcing this incredibly helpful tokenizer.
Following the recommendations from Google\u2019s research on `),Je=a(z,"A",{href:!0,rel:!0});var Ma=s(Je);Eo=l(Ma,"PaLM"),Ma.forEach(o),Po=l(z,", we have removed bias parameters from transformer blocks, achieving better model performance. Please refer "),we=a(z,"A",{href:!0,rel:!0});var Ua=s(we);Xo=l(Ua,"this article"),Ua.forEach(o),Co=l(z," in detail."),z.forEach(o),Qt=h(t),U=a(t,"P",{});var E=s(U);zo=l(E,"Development of the model was led by "),ke=a(E,"A",{href:!0,rel:!0});var Na=s(ke);xo=l(Na,"Shinya Otani"),Na.forEach(o),Fo=l(E,", "),Me=a(E,"A",{href:!0,rel:!0});var $a=s(Me);Vo=l($a,"Takayoshi Makabe"),$a.forEach(o),Zo=l(E,", "),Ue=a(E,"A",{href:!0,rel:!0});var Ga=s(Ue);Io=l(Ga,"Anuj Arora"),Ga.forEach(o),Wo=l(E,", and "),Ne=a(E,"A",{href:!0,rel:!0});var ja=s(Ne);Bo=l(ja,"Kyo Hattori"),ja.forEach(o),So=l(E," from "),$e=a(E,"A",{href:!0,rel:!0});var Ea=s($e);Ro=l(Ea,"ABEJA, Inc."),Ea.forEach(o),qo=l(E,". For more information on this model-building activity, please refer "),Ge=a(E,"A",{href:!0,rel:!0});var Pa=s(Ge);Ao=l(Pa,"here (ja)"),Pa.forEach(o),Lo=l(E,"."),E.forEach(o),Ot=h(t),B=a(t,"H3",{class:!0});var co=s(B);te=a(co,"A",{id:!0,class:!0,href:!0});var Xa=s(te);gt=a(Xa,"SPAN",{});var Ca=s(gt);v(je.$$.fragment,Ca),Ca.forEach(o),Xa.forEach(o),Qo=h(co),_t=a(co,"SPAN",{});var za=s(_t);Oo=l(za,"Generation"),za.forEach(o),co.forEach(o),Yt=h(t),oe=a(t,"P",{});var po=s(oe);Yo=l(po,"The "),Tt=a(po,"CODE",{});var xa=s(Tt);Ho=l(xa,"generate()"),xa.forEach(o),Do=l(po," method can be used to generate text using GPT NeoX Japanese model."),po.forEach(o),Ht=h(t),v(Ee.$$.fragment,t),Dt=h(t),S=a(t,"H2",{class:!0});var ho=s(S);ne=a(ho,"A",{id:!0,class:!0,href:!0});var Fa=s(ne);bt=a(Fa,"SPAN",{});var Va=s(bt);v(Pe.$$.fragment,Va),Va.forEach(o),Fa.forEach(o),Ko=h(ho),yt=a(ho,"SPAN",{});var Za=s(yt);en=l(Za,"Documentation resources"),Za.forEach(o),ho.forEach(o),Kt=h(t),De=a(t,"UL",{});var Ia=s(De);vt=a(Ia,"LI",{});var Wa=s(vt);Ke=a(Wa,"A",{href:!0});var Ba=s(Ke);tn=l(Ba,"Causal language modeling task guide"),Ba.forEach(o),Wa.forEach(o),Ia.forEach(o),eo=h(t),R=a(t,"H2",{class:!0});var mo=s(R);ae=a(mo,"A",{id:!0,class:!0,href:!0});var Sa=s(ae);Jt=a(Sa,"SPAN",{});var Ra=s(Jt);v(Xe.$$.fragment,Ra),Ra.forEach(o),Sa.forEach(o),on=h(mo),wt=a(mo,"SPAN",{});var qa=s(wt);nn=l(qa,"GPTNeoXJapaneseConfig"),qa.forEach(o),mo.forEach(o),to=h(t),P=a(t,"DIV",{class:!0});var fe=s(P);v(Ce.$$.fragment,fe),an=h(fe),q=a(fe,"P",{});var st=s(q);sn=l(st,"This is the configuration class to store the configuration of a "),kt=a(st,"CODE",{});var Aa=s(kt);rn=l(Aa,"GPTNeoXModelJapanese"),Aa.forEach(o),ln=l(st,`. It is used to instantiate
a GPTNeoX model according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of the GPTNeoXJapanese
`),ze=a(st,"A",{href:!0,rel:!0});var La=s(ze);dn=l(La,"abeja/gpt-neox-japanese-2.7b"),La.forEach(o),cn=l(st," architecture."),st.forEach(o),pn=h(fe),A=a(fe,"P",{});var rt=s(A);hn=l(rt,"Configuration objects inherit from "),et=a(rt,"A",{href:!0});var Qa=s(et);mn=l(Qa,"PretrainedConfig"),Qa.forEach(o),un=l(rt,` and can be used to control the model outputs. Read the
documentation from `),tt=a(rt,"A",{href:!0});var Oa=s(tt);fn=l(Oa,"PretrainedConfig"),Oa.forEach(o),gn=l(rt," for more information. Default configs is set as 2.7B model"),rt.forEach(o),_n=h(fe),v(se.$$.fragment,fe),fe.forEach(o),oo=h(t),L=a(t,"H2",{class:!0});var uo=s(L);re=a(uo,"A",{id:!0,class:!0,href:!0});var Ya=s(re);Mt=a(Ya,"SPAN",{});var Ha=s(Mt);v(xe.$$.fragment,Ha),Ha.forEach(o),Ya.forEach(o),Tn=h(uo),Ut=a(uo,"SPAN",{});var Da=s(Ut);bn=l(Da,"GPTNeoXJapaneseTokenizer"),Da.forEach(o),uo.forEach(o),no=h(t),$=a(t,"DIV",{class:!0});var I=s($);v(Fe.$$.fragment,I),yn=h(I),Q=a(I,"P",{});var lt=s(Q);vn=l(lt,"This tokenizer inherits from "),ot=a(lt,"A",{href:!0});var Ka=s(ot);Jn=l(Ka,"PreTrainedTokenizer"),Ka.forEach(o),wn=l(lt,` and is based on Japanese special Sub-Word-Encoding that is
used in this repository (`),Ve=a(lt,"A",{href:!0,rel:!0});var es=s(Ve);kn=l(es,"https://github.com/tanreinama/Japanese-BPEEncoder_V2"),es.forEach(o),Mn=l(lt,`). Check the repository for details.
Japanese has a relatively large vocabulary and there is no separation between words. Furthermore, the language is a
combination of hiragana, katakana, and kanji, and variants such as \u201C1\u201D and \u201C\u2460\u201D are often used. In order to cope
with these, this tokenizer has the following features`),lt.forEach(o),Un=h(I),G=a(I,"UL",{});var x=s(G);Nt=a(x,"LI",{});var ts=s(Nt);Nn=l(ts,"Subword-by-subword segmentation, which is intermediate between byte strings and morphological analysis."),ts.forEach(o),$n=h(x),$t=a(x,"LI",{});var os=s($t);Gn=l(os,`BPEs are created for each Kanji, Hiragana, and Katakana character, and there are no BPEs that cross character
types, such as Kanji + Hiragana or Hiragana + Katakana.`),os.forEach(o),jn=h(x),Gt=a(x,"LI",{});var ns=s(Gt);En=l(ns,"All-byte encoding that does not require <unk>."),ns.forEach(o),Pn=h(x),jt=a(x,"LI",{});var as=s(jt);Xn=l(as,"Independent of UTF codes such as 2-byte and 3-byte characters"),as.forEach(o),Cn=h(x),Et=a(x,"LI",{});var ss=s(Et);zn=l(ss,"Conversion of heterographs to the same token_id"),ss.forEach(o),xn=h(x),Pt=a(x,"LI",{});var rs=s(Pt);Fn=l(rs,"Emoji and Emoticon are grouped into 12 types as special tags."),rs.forEach(o),x.forEach(o),Vn=h(I),v(le.$$.fragment,I),Zn=h(I),ie=a(I,"DIV",{class:!0});var fo=s(ie);v(Ze.$$.fragment,fo),In=h(fo),Xt=a(fo,"P",{});var ls=s(Xt);Wn=l(ls,"Converts a sequence of tokens (string) in a single string."),ls.forEach(o),fo.forEach(o),I.forEach(o),ao=h(t),O=a(t,"H2",{class:!0});var go=s(O);de=a(go,"A",{id:!0,class:!0,href:!0});var is=s(de);Ct=a(is,"SPAN",{});var ds=s(Ct);v(Ie.$$.fragment,ds),ds.forEach(o),is.forEach(o),Bn=h(go),zt=a(go,"SPAN",{});var cs=s(zt);Sn=l(cs,"GPTNeoXJapaneseModel"),cs.forEach(o),go.forEach(o),so=h(t),V=a(t,"DIV",{class:!0});var it=s(V);v(We.$$.fragment,it),Rn=h(it),Be=a(it,"P",{});var _o=s(Be);qn=l(_o,`The bare GPTNeoXJapanese Model transformer outputting raw hidden-states without any specific head on top.
This model is a PyTorch `),Se=a(_o,"A",{href:!0,rel:!0});var ps=s(Se);An=l(ps,"torch.nn.Module"),ps.forEach(o),Ln=l(_o,` sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),_o.forEach(o),Qn=h(it),X=a(it,"DIV",{class:!0});var ge=s(X);v(Re.$$.fragment,ge),On=h(ge),Y=a(ge,"P",{});var dt=s(Y);Yn=l(dt,"The "),nt=a(dt,"A",{href:!0});var hs=s(nt);Hn=l(hs,"GPTNeoXJapaneseModel"),hs.forEach(o),Dn=l(dt," forward method, overrides the "),xt=a(dt,"CODE",{});var ms=s(xt);Kn=l(ms,"__call__"),ms.forEach(o),ea=l(dt," special method."),dt.forEach(o),ta=h(ge),v(ce.$$.fragment,ge),oa=h(ge),v(pe.$$.fragment,ge),ge.forEach(o),it.forEach(o),ro=h(t),H=a(t,"H2",{class:!0});var To=s(H);he=a(To,"A",{id:!0,class:!0,href:!0});var us=s(he);Ft=a(us,"SPAN",{});var fs=s(Ft);v(qe.$$.fragment,fs),fs.forEach(o),us.forEach(o),na=h(To),Vt=a(To,"SPAN",{});var gs=s(Vt);aa=l(gs,"GPTNeoXJapaneseForCausalLM"),gs.forEach(o),To.forEach(o),lo=h(t),Z=a(t,"DIV",{class:!0});var ct=s(Z);v(Ae.$$.fragment,ct),sa=h(ct),D=a(ct,"P",{});var pt=s(D);ra=l(pt,"GPTNeoXJapanese Model with a "),Zt=a(pt,"CODE",{});var _s=s(Zt);la=l(_s,"language modeling"),_s.forEach(o),ia=l(pt,` head on top for Classifier Model fine-tuning.
This model is a PyTorch `),Le=a(pt,"A",{href:!0,rel:!0});var Ts=s(Le);da=l(Ts,"torch.nn.Module"),Ts.forEach(o),ca=l(pt,` sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),pt.forEach(o),pa=h(ct),C=a(ct,"DIV",{class:!0});var _e=s(C);v(Qe.$$.fragment,_e),ha=h(_e),K=a(_e,"P",{});var ht=s(K);ma=l(ht,"The "),at=a(ht,"A",{href:!0});var bs=s(at);ua=l(bs,"GPTNeoXJapaneseForCausalLM"),bs.forEach(o),fa=l(ht," forward method, overrides the "),It=a(ht,"CODE",{});var ys=s(It);ga=l(ys,"__call__"),ys.forEach(o),_a=l(ht," special method."),ht.forEach(o),Ta=h(_e),v(me.$$.fragment,_e),ba=h(_e),v(ue.$$.fragment,_e),_e.forEach(o),ct.forEach(o),this.h()},h(){i(c,"name","hf:doc:metadata"),i(c,"content",JSON.stringify(Cs)),i(f,"id","gptneoxjapanese"),i(f,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(f,"href","#gptneoxjapanese"),i(u,"class","relative group"),i(ee,"id","overview"),i(ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(ee,"href","#overview"),i(W,"class","relative group"),i(ye,"href","https://github.com/EleutherAI/gpt-neox"),i(ye,"rel","nofollow"),i(ve,"href","https://github.com/tanreinama/Japanese-BPEEncoder_V2"),i(ve,"rel","nofollow"),i(Je,"href","https://ai.googleblog.com/2022/04/pathways-language-model-palm-scaling-to.html"),i(Je,"rel","nofollow"),i(we,"href","https://medium.com/ml-abeja/training-a-better-gpt-2-93b157662ae4"),i(we,"rel","nofollow"),i(ke,"href","https://github.com/SO0529"),i(ke,"rel","nofollow"),i(Me,"href","https://github.com/spider-man-tm"),i(Me,"rel","nofollow"),i(Ue,"href","https://github.com/Anuj040"),i(Ue,"rel","nofollow"),i(Ne,"href","https://github.com/go5paopao"),i(Ne,"rel","nofollow"),i($e,"href","https://www.abejainc.com/"),i($e,"rel","nofollow"),i(Ge,"href","https://tech-blog.abeja.asia/entry/abeja-gpt-project-202207"),i(Ge,"rel","nofollow"),i(te,"id","generation"),i(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(te,"href","#generation"),i(B,"class","relative group"),i(ne,"id","documentation-resources"),i(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(ne,"href","#documentation-resources"),i(S,"class","relative group"),i(Ke,"href","../tasks/language_modeling"),i(ae,"id","transformers.GPTNeoXJapaneseConfig"),i(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(ae,"href","#transformers.GPTNeoXJapaneseConfig"),i(R,"class","relative group"),i(ze,"href","https://huggingface.co/abeja/gpt-neox-japanese-2.7b"),i(ze,"rel","nofollow"),i(et,"href","/docs/transformers/v4.31.0/en/main_classes/configuration#transformers.PretrainedConfig"),i(tt,"href","/docs/transformers/v4.31.0/en/main_classes/configuration#transformers.PretrainedConfig"),i(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(re,"id","transformers.GPTNeoXJapaneseTokenizer"),i(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(re,"href","#transformers.GPTNeoXJapaneseTokenizer"),i(L,"class","relative group"),i(ot,"href","/docs/transformers/v4.31.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizer"),i(Ve,"href","https://github.com/tanreinama/Japanese-BPEEncoder_V2"),i(Ve,"rel","nofollow"),i(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(de,"id","transformers.GPTNeoXJapaneseModel"),i(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(de,"href","#transformers.GPTNeoXJapaneseModel"),i(O,"class","relative group"),i(Se,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),i(Se,"rel","nofollow"),i(nt,"href","/docs/transformers/v4.31.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseModel"),i(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(he,"id","transformers.GPTNeoXJapaneseForCausalLM"),i(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(he,"href","#transformers.GPTNeoXJapaneseForCausalLM"),i(H,"class","relative group"),i(Le,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),i(Le,"rel","nofollow"),i(at,"href","/docs/transformers/v4.31.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseForCausalLM"),i(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,m){e(document.head,c),g(t,T,m),g(t,u,m),e(u,f),e(f,b),J(d,b,null),e(u,_),e(u,F),e(F,vo),g(t,At,m),g(t,W,m),e(W,ee),e(ee,mt),J(be,mt,null),e(W,Jo),e(W,ut),e(ut,wo),g(t,Lt,m),g(t,N,m),e(N,ko),e(N,ye),e(ye,Mo),e(N,Uo),e(N,ve),e(ve,No),e(N,$o),e(N,ft),e(ft,Go),e(N,jo),e(N,Je),e(Je,Eo),e(N,Po),e(N,we),e(we,Xo),e(N,Co),g(t,Qt,m),g(t,U,m),e(U,zo),e(U,ke),e(ke,xo),e(U,Fo),e(U,Me),e(Me,Vo),e(U,Zo),e(U,Ue),e(Ue,Io),e(U,Wo),e(U,Ne),e(Ne,Bo),e(U,So),e(U,$e),e($e,Ro),e(U,qo),e(U,Ge),e(Ge,Ao),e(U,Lo),g(t,Ot,m),g(t,B,m),e(B,te),e(te,gt),J(je,gt,null),e(B,Qo),e(B,_t),e(_t,Oo),g(t,Yt,m),g(t,oe,m),e(oe,Yo),e(oe,Tt),e(Tt,Ho),e(oe,Do),g(t,Ht,m),J(Ee,t,m),g(t,Dt,m),g(t,S,m),e(S,ne),e(ne,bt),J(Pe,bt,null),e(S,Ko),e(S,yt),e(yt,en),g(t,Kt,m),g(t,De,m),e(De,vt),e(vt,Ke),e(Ke,tn),g(t,eo,m),g(t,R,m),e(R,ae),e(ae,Jt),J(Xe,Jt,null),e(R,on),e(R,wt),e(wt,nn),g(t,to,m),g(t,P,m),J(Ce,P,null),e(P,an),e(P,q),e(q,sn),e(q,kt),e(kt,rn),e(q,ln),e(q,ze),e(ze,dn),e(q,cn),e(P,pn),e(P,A),e(A,hn),e(A,et),e(et,mn),e(A,un),e(A,tt),e(tt,fn),e(A,gn),e(P,_n),J(se,P,null),g(t,oo,m),g(t,L,m),e(L,re),e(re,Mt),J(xe,Mt,null),e(L,Tn),e(L,Ut),e(Ut,bn),g(t,no,m),g(t,$,m),J(Fe,$,null),e($,yn),e($,Q),e(Q,vn),e(Q,ot),e(ot,Jn),e(Q,wn),e(Q,Ve),e(Ve,kn),e(Q,Mn),e($,Un),e($,G),e(G,Nt),e(Nt,Nn),e(G,$n),e(G,$t),e($t,Gn),e(G,jn),e(G,Gt),e(Gt,En),e(G,Pn),e(G,jt),e(jt,Xn),e(G,Cn),e(G,Et),e(Et,zn),e(G,xn),e(G,Pt),e(Pt,Fn),e($,Vn),J(le,$,null),e($,Zn),e($,ie),J(Ze,ie,null),e(ie,In),e(ie,Xt),e(Xt,Wn),g(t,ao,m),g(t,O,m),e(O,de),e(de,Ct),J(Ie,Ct,null),e(O,Bn),e(O,zt),e(zt,Sn),g(t,so,m),g(t,V,m),J(We,V,null),e(V,Rn),e(V,Be),e(Be,qn),e(Be,Se),e(Se,An),e(Be,Ln),e(V,Qn),e(V,X),J(Re,X,null),e(X,On),e(X,Y),e(Y,Yn),e(Y,nt),e(nt,Hn),e(Y,Dn),e(Y,xt),e(xt,Kn),e(Y,ea),e(X,ta),J(ce,X,null),e(X,oa),J(pe,X,null),g(t,ro,m),g(t,H,m),e(H,he),e(he,Ft),J(qe,Ft,null),e(H,na),e(H,Vt),e(Vt,aa),g(t,lo,m),g(t,Z,m),J(Ae,Z,null),e(Z,sa),e(Z,D),e(D,ra),e(D,Zt),e(Zt,la),e(D,ia),e(D,Le),e(Le,da),e(D,ca),e(Z,pa),e(Z,C),J(Qe,C,null),e(C,ha),e(C,K),e(K,ma),e(K,at),e(at,ua),e(K,fa),e(K,It),e(It,ga),e(K,_a),e(C,Ta),J(me,C,null),e(C,ba),J(ue,C,null),io=!0},p(t,[m]){const Oe={};m&2&&(Oe.$$scope={dirty:m,ctx:t}),se.$set(Oe);const Wt={};m&2&&(Wt.$$scope={dirty:m,ctx:t}),le.$set(Wt);const Bt={};m&2&&(Bt.$$scope={dirty:m,ctx:t}),ce.$set(Bt);const St={};m&2&&(St.$$scope={dirty:m,ctx:t}),pe.$set(St);const Ye={};m&2&&(Ye.$$scope={dirty:m,ctx:t}),me.$set(Ye);const Rt={};m&2&&(Rt.$$scope={dirty:m,ctx:t}),ue.$set(Rt)},i(t){io||(w(d.$$.fragment,t),w(be.$$.fragment,t),w(je.$$.fragment,t),w(Ee.$$.fragment,t),w(Pe.$$.fragment,t),w(Xe.$$.fragment,t),w(Ce.$$.fragment,t),w(se.$$.fragment,t),w(xe.$$.fragment,t),w(Fe.$$.fragment,t),w(le.$$.fragment,t),w(Ze.$$.fragment,t),w(Ie.$$.fragment,t),w(We.$$.fragment,t),w(Re.$$.fragment,t),w(ce.$$.fragment,t),w(pe.$$.fragment,t),w(qe.$$.fragment,t),w(Ae.$$.fragment,t),w(Qe.$$.fragment,t),w(me.$$.fragment,t),w(ue.$$.fragment,t),io=!0)},o(t){k(d.$$.fragment,t),k(be.$$.fragment,t),k(je.$$.fragment,t),k(Ee.$$.fragment,t),k(Pe.$$.fragment,t),k(Xe.$$.fragment,t),k(Ce.$$.fragment,t),k(se.$$.fragment,t),k(xe.$$.fragment,t),k(Fe.$$.fragment,t),k(le.$$.fragment,t),k(Ze.$$.fragment,t),k(Ie.$$.fragment,t),k(We.$$.fragment,t),k(Re.$$.fragment,t),k(ce.$$.fragment,t),k(pe.$$.fragment,t),k(qe.$$.fragment,t),k(Ae.$$.fragment,t),k(Qe.$$.fragment,t),k(me.$$.fragment,t),k(ue.$$.fragment,t),io=!1},d(t){o(c),t&&o(T),t&&o(u),M(d),t&&o(At),t&&o(W),M(be),t&&o(Lt),t&&o(N),t&&o(Qt),t&&o(U),t&&o(Ot),t&&o(B),M(je),t&&o(Yt),t&&o(oe),t&&o(Ht),M(Ee,t),t&&o(Dt),t&&o(S),M(Pe),t&&o(Kt),t&&o(De),t&&o(eo),t&&o(R),M(Xe),t&&o(to),t&&o(P),M(Ce),M(se),t&&o(oo),t&&o(L),M(xe),t&&o(no),t&&o($),M(Fe),M(le),M(Ze),t&&o(ao),t&&o(O),M(Ie),t&&o(so),t&&o(V),M(We),M(Re),M(ce),M(pe),t&&o(ro),t&&o(H),M(qe),t&&o(lo),t&&o(Z),M(Ae),M(Qe),M(me),M(ue)}}}const Cs={local:"gptneoxjapanese",sections:[{local:"overview",sections:[{local:"generation",title:"Generation"}],title:"Overview"},{local:"documentation-resources",title:"Documentation resources"},{local:"transformers.GPTNeoXJapaneseConfig",title:"GPTNeoXJapaneseConfig"},{local:"transformers.GPTNeoXJapaneseTokenizer",title:"GPTNeoXJapaneseTokenizer"},{local:"transformers.GPTNeoXJapaneseModel",title:"GPTNeoXJapaneseModel"},{local:"transformers.GPTNeoXJapaneseForCausalLM",title:"GPTNeoXJapaneseForCausalLM"}],title:"GPT-NeoX-Japanese"};function zs(j){return Us(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Bs extends Js{constructor(c){super();ws(this,c,zs,Xs,ks,{})}}export{Bs as default,Cs as metadata};
