import{S as dl,i as cl,s as hl,e as a,k as c,w as y,t as r,M as ml,c as s,d as n,m as h,a as o,x as b,h as l,b as u,G as t,g as f,y as J,q as M,o as U,B as k,v as ul,L as at}from"../../chunks/vendor-hf-doc-builder.js";import{T as pl}from"../../chunks/Tip-hf-doc-builder.js";import{D as Ue}from"../../chunks/Docstring-hf-doc-builder.js";import{C as ke}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Q}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as nt}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function fl(G){let p,T,m,g,j;return g=new ke({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVFNhbkphcGFuZXNlVG9rZW5pemVyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwR1BUU2FuSmFwYW5lc2VUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMlRhbnJlaSUyRkdQVFNBTi1qYXBhbmVzZSUyMiklMEElMjMlMjBZb3UlMjBjYW4lMjBjb25maXJtJTIwYm90aCUyMCVFNiU4NSVCNiVFNSVCRiU5QyUyMGFuZCUyMCVFNiU4NSVCNiVFNiU4NyU4OSUyMGFyZSUyMGVuY29kZWQlMjB0byUyMDE3NzUwJTBBdG9rZW5pemVyKCUyMiVFNSU5MCVCRSVFOCVCQyVBOSVFMyU4MSVBRiVFNyU4QyVBQiVFMyU4MSVBNyVFMyU4MSU4MiVFMyU4MiU4QiVGMCU5RiU5MCVBRiVFMyU4MCU4MiVFNSVBRSU5RiVFMyU4MSVBRiVFNiU4NSVCNiVFNSVCRiU5QyglRTYlODUlQjYlRTYlODclODkpJUU1JUE0JUE3JUU1JUFEJUE2JUU1JTg3JUJBJUU4JUJBJUFCJTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUwQSUwQSUyMyUyMEJvdGglMjAlRTYlODUlQjYlRTUlQkYlOUMlMjBhbmQlMjAlRTYlODUlQjYlRTYlODclODklMjBhcmUlMjBkZWNvZGVkJTIwdG8lMjAlRTYlODUlQjYlRTUlQkYlOUMlMEF0b2tlbml6ZXIuZGVjb2RlKHRva2VuaXplciglMjIlRTUlOTAlQkUlRTglQkMlQTklRTMlODElQUYlRTclOEMlQUIlRTMlODElQTclRTMlODElODIlRTMlODIlOEIlRjAlOUYlOTAlQUYlRTMlODAlODIlRTUlQUUlOUYlRTMlODElQUYlRTYlODUlQjYlRTUlQkYlOUMoJUU2JTg1JUI2JUU2JTg3JTg5KSVFNSVBNCVBNyVFNSVBRCVBNiVFNSU4NyVCQSVFOCVCQSVBQiUyMiklNUIlMjJpbnB1dF9pZHMlMjIlNUQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSanJapaneseTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTSanJapaneseTokenizer.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># You can confirm both \u6176\u5FDC and \u6176\u61C9 are encoded to 17750</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002\u5B9F\u306F\u6176\u5FDC(\u6176\u61C9)\u5927\u5B66\u51FA\u8EAB&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[<span class="hljs-number">35993</span>, <span class="hljs-number">35998</span>, <span class="hljs-number">34347</span>, <span class="hljs-number">31459</span>, <span class="hljs-number">30647</span>, <span class="hljs-number">31448</span>, <span class="hljs-number">25</span>, <span class="hljs-number">30659</span>, <span class="hljs-number">35729</span>, <span class="hljs-number">35676</span>, <span class="hljs-number">32417</span>, <span class="hljs-number">30647</span>, <span class="hljs-number">17750</span>, <span class="hljs-number">35589</span>, <span class="hljs-number">17750</span>, <span class="hljs-number">35590</span>, <span class="hljs-number">321</span>, <span class="hljs-number">1281</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Both \u6176\u5FDC and \u6176\u61C9 are decoded to \u6176\u5FDC</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(tokenizer(<span class="hljs-string">&quot;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002\u5B9F\u306F\u6176\u5FDC(\u6176\u61C9)\u5927\u5B66\u51FA\u8EAB&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002\u5B9F\u306F\u6176\u5FDC(\u6176\u5FDC)\u5927\u5B66\u51FA\u8EAB&#x27;</span>`}}),{c(){p=a("p"),T=r("Example:"),m=c(),y(g.$$.fragment)},l(i){p=s(i,"P",{});var _=o(p);T=l(_,"Example:"),_.forEach(n),m=h(i),b(g.$$.fragment,i)},m(i,_){f(i,p,_),t(p,T),f(i,m,_),J(g,i,_),j=!0},p:at,i(i){j||(M(g.$$.fragment,i),j=!0)},o(i){U(g.$$.fragment,i),j=!1},d(i){i&&n(p),i&&n(m),k(g,i)}}}function gl(G){let p,T,m,g,j;return g=new ke({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVFNhbkphcGFuZXNlVG9rZW5pemVyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwR1BUU2FuSmFwYW5lc2VUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMlRhbnJlaSUyRkdQVFNBTi1qYXBhbmVzZSUyMiklMEF0b2tlbml6ZXIoJTIyJUU1JUFFJTlGJUUzJTgxJUFGJUU2JTg1JUI2JUU1JUJGJTlDKCVFNiU4NSVCNiVFNiU4NyU4OSklRTUlQTQlQTclRTUlQUQlQTYlRTUlODclQkElRTglQkElQUIlMjIlMkMlMjBwcmVmaXhfdGV4dCUzRCUyMiVFNSU5MCVCRSVFOCVCQyVBOSVFMyU4MSVBRiVFNyU4QyVBQiVFMyU4MSVBNyVFMyU4MSU4MiVFMyU4MiU4QiVGMCU5RiU5MCVBRiVFMyU4MCU4MiUyMiklNUIlMjJpbnB1dF9pZHMlMjIlNUQlMEElMEElMjMlMjBNYXNrJTIwZm9yJTIwUHJlZml4LUxNJTIwaW5wdXRzJTBBdG9rZW5pemVyKCUyMiVFNSVBRSU5RiVFMyU4MSVBRiVFNiU4NSVCNiVFNSVCRiU5QyglRTYlODUlQjYlRTYlODclODkpJUU1JUE0JUE3JUU1JUFEJUE2JUU1JTg3JUJBJUU4JUJBJUFCJTIyJTJDJTIwcHJlZml4X3RleHQlM0QlMjIlRTUlOTAlQkUlRTglQkMlQTklRTMlODElQUYlRTclOEMlQUIlRTMlODElQTclRTMlODElODIlRTMlODIlOEIlRjAlOUYlOTAlQUYlRTMlODAlODIlMjIpJTVCJTIydG9rZW5fdHlwZV9pZHMlMjIlNUQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSanJapaneseTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTSanJapaneseTokenizer.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;\u5B9F\u306F\u6176\u5FDC(\u6176\u61C9)\u5927\u5B66\u51FA\u8EAB&quot;</span>, prefix_text=<span class="hljs-string">&quot;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[<span class="hljs-number">35993</span>, <span class="hljs-number">34347</span>, <span class="hljs-number">31459</span>, <span class="hljs-number">30647</span>, <span class="hljs-number">31448</span>, <span class="hljs-number">25</span>, <span class="hljs-number">30659</span>, <span class="hljs-number">35729</span>, <span class="hljs-number">35676</span>, <span class="hljs-number">35998</span>, <span class="hljs-number">32417</span>, <span class="hljs-number">30647</span>, <span class="hljs-number">17750</span>, <span class="hljs-number">35589</span>, <span class="hljs-number">17750</span>, <span class="hljs-number">35590</span>, <span class="hljs-number">321</span>, <span class="hljs-number">1281</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Mask for Prefix-LM inputs</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;\u5B9F\u306F\u6176\u5FDC(\u6176\u61C9)\u5927\u5B66\u51FA\u8EAB&quot;</span>, prefix_text=<span class="hljs-string">&quot;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002&quot;</span>)[<span class="hljs-string">&quot;token_type_ids&quot;</span>]
[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]`}}),{c(){p=a("p"),T=r("Example for Prefix-LM:"),m=c(),y(g.$$.fragment)},l(i){p=s(i,"P",{});var _=o(p);T=l(_,"Example for Prefix-LM:"),_.forEach(n),m=h(i),b(g.$$.fragment,i)},m(i,_){f(i,p,_),t(p,T),f(i,m,_),J(g,i,_),j=!0},p:at,i(i){j||(M(g.$$.fragment,i),j=!0)},o(i){U(g.$$.fragment,i),j=!1},d(i){i&&n(p),i&&n(m),k(g,i)}}}function _l(G){let p,T,m,g,j;return g=new ke({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVFNhbkphcGFuZXNlVG9rZW5pemVyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwR1BUU2FuSmFwYW5lc2VUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMlRhbnJlaSUyRkdQVFNBTi1qYXBhbmVzZSUyMiklMEF0b2tlbml6ZXIoJTVCJTVCJTIyJUU2JUFEJUE2JUU3JTk0JUIwJUU0JUJGJUExJUU3JThFJTg0JTIyJTJDJTIwJTIyJUUzJTgxJUFGJUUzJTgwJTgxJTIyJTVEJTJDJTIwJTVCJTIyJUU3JUI5JTk0JUU3JTk0JUIwJUU0JUJGJUExJUU5JTk1JUI3JTIyJTJDJTIwJTIyJUUzJTgxJUFFJUU5JTg1JThEJUU0JUI4JThCJUUzJTgxJUFFJUUzJTgwJTgxJTIyJTVEJTVEJTJDJTIwcGFkZGluZyUzRFRydWUpJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTBBJTBBJTIzJTIwTWFzayUyMGZvciUyMFByZWZpeC1MTSUyMGlucHV0cyUwQXRva2VuaXplciglNUIlNUIlMjIlRTYlQUQlQTYlRTclOTQlQjAlRTQlQkYlQTElRTclOEUlODQlMjIlMkMlMjAlMjIlRTMlODElQUYlRTMlODAlODElMjIlNUQlMkMlMjAlNUIlMjIlRTclQjklOTQlRTclOTQlQjAlRTQlQkYlQTElRTklOTUlQjclMjIlMkMlMjAlMjIlRTMlODElQUUlRTklODUlOEQlRTQlQjglOEIlRTMlODElQUUlRTMlODAlODElMjIlNUQlNUQlMkMlMjBwYWRkaW5nJTNEVHJ1ZSklNUIlMjJ0b2tlbl90eXBlX2lkcyUyMiU1RCUwQSUwQSUyMyUyME1hc2slMjBmb3IlMjBwYWRkaW5nJTBBdG9rZW5pemVyKCU1QiU1QiUyMiVFNiVBRCVBNiVFNyU5NCVCMCVFNCVCRiVBMSVFNyU4RSU4NCUyMiUyQyUyMCUyMiVFMyU4MSVBRiVFMyU4MCU4MSUyMiU1RCUyQyUyMCU1QiUyMiVFNyVCOSU5NCVFNyU5NCVCMCVFNCVCRiVBMSVFOSU5NSVCNyUyMiUyQyUyMCUyMiVFMyU4MSVBRSVFOSU4NSU4RCVFNCVCOCU4QiVFMyU4MSVBRSVFMyU4MCU4MSUyMiU1RCU1RCUyQyUyMHBhZGRpbmclM0RUcnVlKSU1QiUyMmF0dGVudGlvbl9tYXNrJTIyJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSanJapaneseTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTSanJapaneseTokenizer.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer([[<span class="hljs-string">&quot;\u6B66\u7530\u4FE1\u7384&quot;</span>, <span class="hljs-string">&quot;\u306F\u3001&quot;</span>], [<span class="hljs-string">&quot;\u7E54\u7530\u4FE1\u9577&quot;</span>, <span class="hljs-string">&quot;\u306E\u914D\u4E0B\u306E\u3001&quot;</span>]], padding=<span class="hljs-literal">True</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[[<span class="hljs-number">35993</span>, <span class="hljs-number">8640</span>, <span class="hljs-number">25948</span>, <span class="hljs-number">35998</span>, <span class="hljs-number">30647</span>, <span class="hljs-number">35675</span>, <span class="hljs-number">35999</span>, <span class="hljs-number">35999</span>], [<span class="hljs-number">35993</span>, <span class="hljs-number">10382</span>, <span class="hljs-number">9868</span>, <span class="hljs-number">35998</span>, <span class="hljs-number">30646</span>, <span class="hljs-number">9459</span>, <span class="hljs-number">30646</span>, <span class="hljs-number">35675</span>]]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Mask for Prefix-LM inputs</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer([[<span class="hljs-string">&quot;\u6B66\u7530\u4FE1\u7384&quot;</span>, <span class="hljs-string">&quot;\u306F\u3001&quot;</span>], [<span class="hljs-string">&quot;\u7E54\u7530\u4FE1\u9577&quot;</span>, <span class="hljs-string">&quot;\u306E\u914D\u4E0B\u306E\u3001&quot;</span>]], padding=<span class="hljs-literal">True</span>)[<span class="hljs-string">&quot;token_type_ids&quot;</span>]
[[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Mask for padding</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer([[<span class="hljs-string">&quot;\u6B66\u7530\u4FE1\u7384&quot;</span>, <span class="hljs-string">&quot;\u306F\u3001&quot;</span>], [<span class="hljs-string">&quot;\u7E54\u7530\u4FE1\u9577&quot;</span>, <span class="hljs-string">&quot;\u306E\u914D\u4E0B\u306E\u3001&quot;</span>]], padding=<span class="hljs-literal">True</span>)[<span class="hljs-string">&quot;attention_mask&quot;</span>]
[[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]`}}),{c(){p=a("p"),T=r("Example for batch encode:"),m=c(),y(g.$$.fragment)},l(i){p=s(i,"P",{});var _=o(p);T=l(_,"Example for batch encode:"),_.forEach(n),m=h(i),b(g.$$.fragment,i)},m(i,_){f(i,p,_),t(p,T),f(i,m,_),J(g,i,_),j=!0},p:at,i(i){j||(M(g.$$.fragment,i),j=!0)},o(i){U(g.$$.fragment,i),j=!1},d(i){i&&n(p),i&&n(m),k(g,i)}}}function Tl(G){let p,T,m,g,j;return g=new ke({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVFNhbkphcGFuZXNlVG9rZW5pemVyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwR1BUU2FuSmFwYW5lc2VUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMlRhbnJlaSUyRkdQVFNBTi1qYXBhbmVzZSUyMiklMEF4X3Rva2VuJTIwJTNEJTIwdG9rZW5pemVyKCUyMiVFRiVCRCVCMSVFRiVCRCVCMiVFRiVCRCVCMyVFRiVCRCVCNCUyMiklMEElMjMlMjBpbnB1dF9pZHMlM0ElMjAlMjAlMjAlMjAlMjAlMjAlN0MlMjBTT1QlMjAlN0MlMjBTRUclMjAlN0MlMjAlRUYlQkQlQjElMjAlN0MlMjAlRUYlQkQlQjIlMjAlN0MlMjAlRUYlQkQlQjMlMjAlN0MlMjAlRUYlQkQlQjQlMjAlN0MlMEElMjMlMjB0b2tlbl90eXBlX2lkcyUzQSUyMCU3QyUyMDElMjAlMjAlMjAlN0MlMjAwJTIwJTIwJTIwJTdDJTIwMCUyMCU3QyUyMDAlMjAlN0MlMjAwJTIwJTdDJTIwMCUyMCU3QyUwQSUwQXhfdG9rZW4lMjAlM0QlMjB0b2tlbml6ZXIoJTIyJTIyJTJDJTIwcHJlZml4X3RleHQlM0QlMjIlRUYlQkQlQjElRUYlQkQlQjIlRUYlQkQlQjMlRUYlQkQlQjQlMjIpJTBBJTIzJTIwaW5wdXRfaWRzJTNBJTIwJTIwJTIwJTIwJTIwJTIwJTdDJTIwU09UJTIwJTdDJTIwJUVGJUJEJUIxJTIwJTdDJTIwJUVGJUJEJUIyJTIwJTdDJTIwJUVGJUJEJUIzJTIwJTdDJTIwJUVGJUJEJUI0JTIwJTdDJTIwU0VHJTIwJTdDJTBBJTIzJTIwdG9rZW5fdHlwZV9pZHMlM0ElMjAlN0MlMjAxJTIwJTIwJTIwJTdDJTIwMSUyMCU3QyUyMDElMjAlN0MlMjAxJTIwJTdDJTIwMSUyMCU3QyUyMDAlMjAlMjAlN0MlMEElMEF4X3Rva2VuJTIwJTNEJTIwdG9rZW5pemVyKCUyMiVFRiVCRCVCMyVFRiVCRCVCNCUyMiUyQyUyMHByZWZpeF90ZXh0JTNEJTIyJUVGJUJEJUIxJUVGJUJEJUIyJTIyKSUwQSUyMyUyMGlucHV0X2lkcyUzQSUyMCUyMCUyMCUyMCUyMCUyMCU3QyUyMFNPVCUyMCU3QyUyMCVFRiVCRCVCMSUyMCU3QyUyMCVFRiVCRCVCMiUyMCU3QyUyMFNFRyUyMCU3QyUyMCVFRiVCRCVCMyUyMCU3QyUyMCVFRiVCRCVCNCUyMCU3QyUwQSUyMyUyMHRva2VuX3R5cGVfaWRzJTNBJTIwJTdDJTIwMSUyMCUyMCUyMCU3QyUyMDElMjAlN0MlMjAxJTIwJTdDJTIwMCUyMCUyMCUyMCU3QyUyMDAlMjAlN0MlMjAwJTIwJTdD",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSanJapaneseTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTSanJapaneseTokenizer.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>x_token = tokenizer(<span class="hljs-string">&quot;\uFF71\uFF72\uFF73\uFF74&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># input_ids:      | SOT | SEG | \uFF71 | \uFF72 | \uFF73 | \uFF74 |</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># token_type_ids: | 1   | 0   | 0 | 0 | 0 | 0 |</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>x_token = tokenizer(<span class="hljs-string">&quot;&quot;</span>, prefix_text=<span class="hljs-string">&quot;\uFF71\uFF72\uFF73\uFF74&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># input_ids:      | SOT | \uFF71 | \uFF72 | \uFF73 | \uFF74 | SEG |</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># token_type_ids: | 1   | 1 | 1 | 1 | 1 | 0  |</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>x_token = tokenizer(<span class="hljs-string">&quot;\uFF73\uFF74&quot;</span>, prefix_text=<span class="hljs-string">&quot;\uFF71\uFF72&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># input_ids:      | SOT | \uFF71 | \uFF72 | SEG | \uFF73 | \uFF74 |</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># token_type_ids: | 1   | 1 | 1 | 0   | 0 | 0 |</span>`}}),{c(){p=a("p"),T=r("Example:"),m=c(),y(g.$$.fragment)},l(i){p=s(i,"P",{});var _=o(p);T=l(_,"Example:"),_.forEach(n),m=h(i),b(g.$$.fragment,i)},m(i,_){f(i,p,_),t(p,T),f(i,m,_),J(g,i,_),j=!0},p:at,i(i){j||(M(g.$$.fragment,i),j=!0)},o(i){U(g.$$.fragment,i),j=!1},d(i){i&&n(p),i&&n(m),k(g,i)}}}function yl(G){let p,T,m,g,j;return{c(){p=a("p"),T=r("Although the recipe for forward pass needs to be defined within this function, one should call the "),m=a("code"),g=r("Module"),j=r(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(i){p=s(i,"P",{});var _=o(p);T=l(_,"Although the recipe for forward pass needs to be defined within this function, one should call the "),m=s(_,"CODE",{});var P=o(m);g=l(P,"Module"),P.forEach(n),j=l(_,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),_.forEach(n)},m(i,_){f(i,p,_),t(p,T),t(p,m),t(m,g),t(p,j)},d(i){i&&n(p)}}}function bl(G){let p,T,m,g,j;return{c(){p=a("p"),T=r("Although the recipe for forward pass needs to be defined within this function, one should call the "),m=a("code"),g=r("Module"),j=r(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(i){p=s(i,"P",{});var _=o(p);T=l(_,"Although the recipe for forward pass needs to be defined within this function, one should call the "),m=s(_,"CODE",{});var P=o(m);g=l(P,"Module"),P.forEach(n),j=l(_,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),_.forEach(n)},m(i,_){f(i,p,_),t(p,T),t(p,m),t(m,g),t(p,j)},d(i){i&&n(p)}}}function Jl(G){let p,T;return p=new ke({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUyQyUyMEF1dG9Ub2tlbml6ZXIlMkMlMjB0cmFpbmVyX3V0aWxzJTBBJTBBZGV2aWNlJTIwJTNEJTIwJTIyY3VkYSUyMiUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJUYW5yZWklMkZHUFRTQU4tamFwYW5lc2UlMjIpLnRvKGRldmljZSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJUYW5yZWklMkZHUFRTQU4tamFwYW5lc2UlMjIpJTBBeF90b2tlbiUyMCUzRCUyMHRva2VuaXplciglMjIlRTclQjklOTQlRTclOTQlQjAlRTQlQkYlQTElRTklOTUlQjclRTMlODElQUYlRTMlODAlODElMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQXRyYWluZXJfdXRpbHMuc2V0X3NlZWQoMzApJTBBaW5wdXRfaWRzJTIwJTNEJTIweF90b2tlbi5pbnB1dF9pZHMudG8oZGV2aWNlKSUwQWdlbl90b2tlbiUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKGlucHV0X2lkcyUyQyUyMG1heF9uZXdfdG9rZW5zJTNENTApJTBBdG9rZW5pemVyLmRlY29kZShnZW5fdG9rZW4lNUIwJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoTokenizer, trainer_utils

<span class="hljs-meta">&gt;&gt;&gt; </span>device = <span class="hljs-string">&quot;cuda&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModel.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>).to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>x_token = tokenizer(<span class="hljs-string">&quot;\u7E54\u7530\u4FE1\u9577\u306F\u3001&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer_utils.set_seed(<span class="hljs-number">30</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = x_token.input_ids.to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>gen_token = model.generate(input_ids, max_new_tokens=<span class="hljs-number">50</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(gen_token[<span class="hljs-number">0</span>])
<span class="hljs-string">&quot;\u7E54\u7530\u4FE1\u9577\u306F\u3001\u653F\u6CBB\u30FB\u8ECD\u4E8B\u306E\u4E2D\u67A2\u307E\u3067\u638C\u63E1\u3057\u305F\u653F\u6CBB\u5BB6\u3067\u3042\u308A\u3001\u65E5\u672C\u53F2\u4E0A\u985E\u3092\u898B\u306A\u3044\u9A5A\u7570\u7684\u306A\u8ECD\u4E8B\u4FB5\u653B\u3092\u7D9A\u3051...&quot;</span>`}}),{c(){y(p.$$.fragment)},l(m){b(p.$$.fragment,m)},m(m,g){J(p,m,g),T=!0},p:at,i(m){T||(M(p.$$.fragment,m),T=!0)},o(m){U(p.$$.fragment,m),T=!1},d(m){k(p,m)}}}function Ml(G){let p,T;return p=new ke({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUyQyUyMEF1dG9Ub2tlbml6ZXIlMkMlMjB0cmFpbmVyX3V0aWxzJTBBJTBBZGV2aWNlJTIwJTNEJTIwJTIyY3VkYSUyMiUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJUYW5yZWklMkZHUFRTQU4tamFwYW5lc2UlMjIpLnRvKGRldmljZSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJUYW5yZWklMkZHUFRTQU4tamFwYW5lc2UlMjIpJTBBeF90b2tlbiUyMCUzRCUyMHRva2VuaXplciglMjIlMjIlMkMlMjBwcmVmaXhfdGV4dCUzRCUyMiVFNyVCOSU5NCVFNyU5NCVCMCVFNCVCRiVBMSVFOSU5NSVCNyVFMyU4MSVBRiVFMyU4MCU4MSUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBdHJhaW5lcl91dGlscy5zZXRfc2VlZCgzMCklMEFpbnB1dF9pZHMlMjAlM0QlMjB4X3Rva2VuLmlucHV0X2lkcy50byhkZXZpY2UpJTBBdG9rZW5fdHlwZV9pZHMlMjAlM0QlMjB4X3Rva2VuLnRva2VuX3R5cGVfaWRzLnRvKGRldmljZSklMEFnZW5fdG9rZW4lMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dF9pZHMlMkMlMjB0b2tlbl90eXBlX2lkcyUzRHRva2VuX3R5cGVfaWRzJTJDJTIwbWF4X25ld190b2tlbnMlM0Q1MCklMEF0b2tlbml6ZXIuZGVjb2RlKGdlbl90b2tlbiU1QjAlNUQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoTokenizer, trainer_utils

<span class="hljs-meta">&gt;&gt;&gt; </span>device = <span class="hljs-string">&quot;cuda&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModel.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>).to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>x_token = tokenizer(<span class="hljs-string">&quot;&quot;</span>, prefix_text=<span class="hljs-string">&quot;\u7E54\u7530\u4FE1\u9577\u306F\u3001&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer_utils.set_seed(<span class="hljs-number">30</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = x_token.input_ids.to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>token_type_ids = x_token.token_type_ids.to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>gen_token = model.generate(input_ids, token_type_ids=token_type_ids, max_new_tokens=<span class="hljs-number">50</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(gen_token[<span class="hljs-number">0</span>])
<span class="hljs-string">&quot;\u7E54\u7530\u4FE1\u9577\u306F\u3001\u653F\u6CBB\u30FB\u5916\u4EA4\u3067\u6570\u3005\u306E\u6226\u679C\u3092\u4E0A\u3052\u308B\u304C\u30011568\u5E74\u304B\u3089\u306F\u3001\u3044\u308F\u3086\u308B\u672C\u80FD\u5BFA\u306E\u5909\u3067\u7D30\u5DDD\u6674\u5143\u306B\u6697\u6BBA\u3055\u308C\u308B...&quot;</span>`}}),{c(){y(p.$$.fragment)},l(m){b(p.$$.fragment,m)},m(m,g){J(p,m,g),T=!0},p:at,i(m){T||(M(p.$$.fragment,m),T=!0)},o(m){U(p.$$.fragment,m),T=!1},d(m){k(p,m)}}}function Ul(G){let p,T;return p=new ke({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUyQyUyMEF1dG9Ub2tlbml6ZXIlMkMlMjB0cmFpbmVyX3V0aWxzJTBBJTBBZGV2aWNlJTIwJTNEJTIwJTIyY3VkYSUyMiUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJUYW5yZWklMkZHUFRTQU4tamFwYW5lc2UlMjIpLnRvKGRldmljZSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJUYW5yZWklMkZHUFRTQU4tamFwYW5lc2UlMjIpJTBBbWFza2VkX3NlbnRlbmNlJTIwJTNEJTIwJTIyJUU2JUFEJUE2JUU3JTk0JUIwJUU0JUJGJUExJUU3JThFJTg0JUUzJTgxJUFGJUUzJTgwJTgxJTNDJTdDaW5wdXRtYXNrJTdDJTNFJUU2JTk5JTgyJUU0JUJCJUEzJUUzJTgzJTk1JUUzJTgyJUExJUUzJTgzJUIzJUUzJTgxJUFBJUUzJTgyJTg5JUUzJTgxJTlDJUUzJTgxJUIyJUU2JThBJUJDJUUzJTgxJTk1JUUzJTgxJTg4JTNDJTdDaW5wdXRtYXNrJTdDJTNFJUUzJTgxJThEJUUzJTgxJTlGJUUzJTgxJTg0JUU1JTkwJThEJUU1JUIwJTg2JUUzJTgxJUFFJUU0JUI4JTgwJUU0JUJBJUJBJUUzJTgwJTgyJTIyJTBBeF90b2tlbiUyMCUzRCUyMHRva2VuaXplciglMjIlMjIlMkMlMjBwcmVmaXhfdGV4dCUzRG1hc2tlZF9zZW50ZW5jZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBdHJhaW5lcl91dGlscy5zZXRfc2VlZCgzMCklMEFpbnB1dF9pZHMlMjAlM0QlMjB4X3Rva2VuLmlucHV0X2lkcy50byhkZXZpY2UpJTBBdG9rZW5fdHlwZV9pZHMlMjAlM0QlMjB4X3Rva2VuLnRva2VuX3R5cGVfaWRzLnRvKGRldmljZSklMEFvdXRfbG1fdG9rZW4lMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dF9pZHMlMkMlMjB0b2tlbl90eXBlX2lkcyUzRHRva2VuX3R5cGVfaWRzJTJDJTIwbWF4X25ld190b2tlbnMlM0Q1MCklMEFvdXRfbWxtX3Rva2VuJTIwJTNEJTIwbW9kZWwoaW5wdXRfaWRzJTJDJTIwdG9rZW5fdHlwZV9pZHMlM0R0b2tlbl90eXBlX2lkcykubG9naXRzLmFyZ21heChheGlzJTNELTEpJTBBdG9rZW5pemVyLmRlY29kZShvdXRfbWxtX3Rva2VuJTVCMCU1RCklMEElMEF0b2tlbml6ZXIuZGVjb2RlKG91dF9sbV90b2tlbiU1QjAlNUQlNUJpbnB1dF9pZHMuc2hhcGUlNUIxJTVEJTIwJTNBJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoTokenizer, trainer_utils

<span class="hljs-meta">&gt;&gt;&gt; </span>device = <span class="hljs-string">&quot;cuda&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModel.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>).to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>masked_sentence = <span class="hljs-string">&quot;\u6B66\u7530\u4FE1\u7384\u306F\u3001&lt;|inputmask|&gt;\u6642\u4EE3\u30D5\u30A1\u30F3\u306A\u3089\u305C\u3072\u62BC\u3055\u3048&lt;|inputmask|&gt;\u304D\u305F\u3044\u540D\u5C06\u306E\u4E00\u4EBA\u3002&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>x_token = tokenizer(<span class="hljs-string">&quot;&quot;</span>, prefix_text=masked_sentence, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer_utils.set_seed(<span class="hljs-number">30</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = x_token.input_ids.to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>token_type_ids = x_token.token_type_ids.to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>out_lm_token = model.generate(input_ids, token_type_ids=token_type_ids, max_new_tokens=<span class="hljs-number">50</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>out_mlm_token = model(input_ids, token_type_ids=token_type_ids).logits.argmax(axis=-<span class="hljs-number">1</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(out_mlm_token[<span class="hljs-number">0</span>])
<span class="hljs-string">&quot;\u6B66\u7530\u4FE1\u7384\u306F\u3001\u6226\u56FD\u6642\u4EE3\u30D5\u30A1\u30F3\u306A\u3089\u305C\u3072\u62BC\u3055\u3048\u3066\u304A\u304D\u305F\u3044\u540D\u5C06\u306E\u4E00\u4EBA\u3002&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(out_lm_token[<span class="hljs-number">0</span>][input_ids.shape[<span class="hljs-number">1</span>] :])
<span class="hljs-string">&quot;\u6B66\u7530\u6C0F\u306E\u4E09\u4EE3\u306B\u6E21\u3063\u305F\u6B66\u7530\u5BB6\u306E\u3072\u3068\u308A\\n\u7532\u6590\u5E02\u306B\u4F4F\u3080\u3001\u65E5\u672C\u53F2\u4E0A\u6700\u5927\u306E\u6226\u56FD\u5927\u540D\u3002...&quot;</span>`}}),{c(){y(p.$$.fragment)},l(m){b(p.$$.fragment,m)},m(m,g){J(p,m,g),T=!0},p:at,i(m){T||(M(p.$$.fragment,m),T=!0)},o(m){U(p.$$.fragment,m),T=!1},d(m){k(p,m)}}}function kl(G){let p,T,m,g,j,i,_,P,ya,Vn,A,ne,vt,je,ba,wt,Ja,xn,st,Ma,$n,ot,Ua,En,q,ae,St,ve,ka,Gt,ja,Cn,se,va,Vt,wa,Sa,Pn,we,Nn,Z,oe,xt,Se,Ga,$t,Va,Fn,rt,xa,Rn,B,re,Et,Ge,$a,Ct,Ea,In,w,Ca,Pt,Pa,Na,Nt,Fa,Ra,Ft,Ia,za,Rt,Qa,Aa,It,qa,Za,zt,Ba,Oa,Qt,Wa,Xa,At,Da,Ya,zn,lt,La,Qn,it,Ha,An,pt,Ka,qn,dt,qt,Zt,Bt,es,Zn,ct,Ot,Wt,Xt,ts,Bn,ht,Dt,Yt,Lt,ns,On,O,le,Ht,Ve,as,Kt,ss,Wn,ie,os,en,rs,ls,Xn,W,pe,tn,xe,is,nn,ps,Dn,N,$e,ds,X,cs,mt,hs,ms,Ee,us,fs,gs,D,_s,ut,Ts,ys,ft,bs,Js,Yn,Y,de,an,Ce,Ms,sn,Us,Ln,S,Pe,ks,on,js,vs,L,rn,ws,Ss,ln,Gs,Vs,Ne,xs,Fe,$s,Es,Cs,ce,Ps,he,Ns,me,Fs,ue,Re,Rs,pn,Is,zs,F,Ie,Qs,dn,As,qs,fe,Hn,H,ge,cn,ze,Zs,hn,Bs,Kn,E,Qe,Os,mn,Ws,Xs,Ae,Ds,qe,Ys,Ls,Hs,Ze,Ks,Be,eo,to,no,R,Oe,ao,K,so,gt,oo,ro,un,lo,io,po,_e,ea,ee,Te,fn,We,co,gn,ho,ta,C,Xe,mo,_n,uo,fo,De,go,Ye,_o,To,yo,Le,bo,He,Jo,Mo,Uo,v,Ke,ko,te,jo,_t,vo,wo,Tn,So,Go,Vo,ye,xo,yn,$o,Eo,bn,Co,Po,be,No,Jn,Fo,Ro,Je,Io,Mn,zo,Qo,Me,na;return i=new Q({}),je=new Q({}),ve=new Q({}),we=new ke({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUyQyUyMEF1dG9Ub2tlbml6ZXIlMEFpbXBvcnQlMjB0b3JjaCUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMlRhbnJlaSUyRkdQVFNBTi1qYXBhbmVzZSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIyVGFucmVpJTJGR1BUU0FOLWphcGFuZXNlJTIyKS5jdWRhKCklMEF4X3RvayUyMCUzRCUyMHRva2VuaXplciglMjIlRTMlODElQUYlRTMlODAlODElMjIlMkMlMjBwcmVmaXhfdGV4dCUzRCUyMiVFNyVCOSU5NCVFNyU5NCVCMCVFNCVCRiVBMSVFOSU5NSVCNyUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBdG9yY2gubWFudWFsX3NlZWQoMCklMEFnZW5fdG9rJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoeF90b2suaW5wdXRfaWRzLmN1ZGEoKSUyQyUyMHRva2VuX3R5cGVfaWRzJTNEeF90b2sudG9rZW5fdHlwZV9pZHMuY3VkYSgpJTJDJTIwbWF4X25ld190b2tlbnMlM0QyMCklMEF0b2tlbml6ZXIuZGVjb2RlKGdlbl90b2slNUIwJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModel.from_pretrained(<span class="hljs-string">&quot;Tanrei/GPTSAN-japanese&quot;</span>).cuda()
<span class="hljs-meta">&gt;&gt;&gt; </span>x_tok = tokenizer(<span class="hljs-string">&quot;\u306F\u3001&quot;</span>, prefix_text=<span class="hljs-string">&quot;\u7E54\u7530\u4FE1\u9577&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>torch.manual_seed(<span class="hljs-number">0</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>gen_tok = model.generate(x_tok.input_ids.cuda(), token_type_ids=x_tok.token_type_ids.cuda(), max_new_tokens=<span class="hljs-number">20</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(gen_tok[<span class="hljs-number">0</span>])
<span class="hljs-string">&#x27;\u7E54\u7530\u4FE1\u9577\u306F\u30012004\u5E74\u306B\u300E\u6226\u56FDBASARA\u300F\u306E\u305F\u3081\u306B\u3001\u8C4A\u81E3\u79C0\u5409&#x27;</span>`}}),Se=new Q({}),Ge=new Q({}),Ve=new Q({}),xe=new Q({}),$e=new Ue({props:{name:"class transformers.GPTSanJapaneseConfig",anchor:"transformers.GPTSanJapaneseConfig",parameters:[{name:"vocab_size",val:" = 36000"},{name:"max_position_embeddings",val:" = 1280"},{name:"d_model",val:" = 1024"},{name:"d_ff",val:" = 8192"},{name:"d_ext",val:" = 4096"},{name:"d_spout",val:" = 128"},{name:"num_switch_layers",val:" = 10"},{name:"num_ext_layers",val:" = 0"},{name:"num_heads",val:" = 16"},{name:"num_experts",val:" = 16"},{name:"expert_capacity",val:" = 128"},{name:"dropout_rate",val:" = 0.0"},{name:"layer_norm_epsilon",val:" = 1e-05"},{name:"router_bias",val:" = False"},{name:"router_jitter_noise",val:" = 0.0"},{name:"router_dtype",val:" = 'float32'"},{name:"router_ignore_padding_tokens",val:" = False"},{name:"output_hidden_states",val:" = False"},{name:"output_attentions",val:" = False"},{name:"initializer_factor",val:" = 0.002"},{name:"output_router_logits",val:" = False"},{name:"use_cache",val:" = True"},{name:"separator_token_id",val:" = 35998"},{name:"pad_token_id",val:" = 35995"},{name:"eos_token_id",val:" = 35999"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTSanJapaneseConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 36000) &#x2014;
Vocabulary size of the GPTSANJapanese model. Defines the number of different tokens that can be represented
by the <code>inputs_ids</code> passed when calling <a href="/docs/transformers/v4.31.0/en/model_doc/gptsan-japanese#transformers.GPTSanJapaneseModel">GPTSanJapaneseModel</a>.`,name:"vocab_size"},{anchor:"transformers.GPTSanJapaneseConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to 1280) &#x2014;
The maximum sequence length that this model might ever be used with. Defaults set this to 1280.`,name:"max_position_embeddings"},{anchor:"transformers.GPTSanJapaneseConfig.d_model",description:`<strong>d_model</strong> (<code>int</code>, <em>optional</em>, defaults to 1024) &#x2014;
Size of the encoder layers and the pooler layer.`,name:"d_model"},{anchor:"transformers.GPTSanJapaneseConfig.d_ff",description:`<strong>d_ff</strong> (<code>int</code>, <em>optional</em>, defaults to 8192) &#x2014;
Size of the intermediate feed forward layer in each <code>SwitchTransformersBlock</code>.`,name:"d_ff"},{anchor:"transformers.GPTSanJapaneseConfig.d_ext",description:`<strong>d_ext</strong> (<code>int</code>, <em>optional</em>, defaults to 4096) &#x2014;
Size of the intermediate feed forward layer in each Extra-layers.`,name:"d_ext"},{anchor:"transformers.GPTSanJapaneseConfig.d_spout",description:`<strong>d_spout</strong> (<code>int</code>, <em>optional</em>, defaults to 128) &#x2014;
Size of the <code>spout</code> vector.`,name:"d_spout"},{anchor:"transformers.GPTSanJapaneseConfig.num_switch_layers",description:`<strong>num_switch_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 10) &#x2014;
Number of layers in the Switch Transformer layer.`,name:"num_switch_layers"},{anchor:"transformers.GPTSanJapaneseConfig.num_ext_layers",description:`<strong>num_ext_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 0) &#x2014;
Number of layers in the Extra-layers.`,name:"num_ext_layers"},{anchor:"transformers.GPTSanJapaneseConfig.num_heads",description:`<strong>num_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 16) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_heads"},{anchor:"transformers.GPTSanJapaneseConfig.num_experts",description:`<strong>num_experts</strong> (<code>int</code>, <em>optional</em>, defaults to 16) &#x2014;
Number of experts for each SwitchTransformer layer.`,name:"num_experts"},{anchor:"transformers.GPTSanJapaneseConfig.expert_capacity",description:`<strong>expert_capacity</strong> (<code>int</code>, <em>optional</em>, defaults to 128) &#x2014;
Number of tokens that can be stored in each expert. If set to 1, the model will behave like a regular
Transformer.`,name:"expert_capacity"},{anchor:"transformers.GPTSanJapaneseConfig.dropout_rate",description:`<strong>dropout_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The ratio for all dropout layers.`,name:"dropout_rate"},{anchor:"transformers.GPTSanJapaneseConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-5) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.GPTSanJapaneseConfig.router_bias",description:`<strong>router_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to add a bias to the router.`,name:"router_bias"},{anchor:"transformers.GPTSanJapaneseConfig.router_jitter_noise",description:`<strong>router_jitter_noise</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
Amount of noise to add to the router. Set it to 0.0 during prediction or set small value (usually 1e-2)
during training.`,name:"router_jitter_noise"},{anchor:"transformers.GPTSanJapaneseConfig.router_dtype",description:`<strong>router_dtype</strong> (<code>str</code>, <em>optional</em>, default to <code>&quot;float32&quot;</code>) &#x2014;
The <code>dtype</code> used for the routers. It is preferable to keep the <code>dtype</code> to <code>&quot;float32&quot;</code> as specified in the
<em>selective precision</em> discussion in <a href="https://arxiv.org/abs/2101.03961" rel="nofollow">the paper</a>.`,name:"router_dtype"},{anchor:"transformers.GPTSanJapaneseConfig.router_ignore_padding_tokens",description:`<strong>router_ignore_padding_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to ignore padding tokens when routing.`,name:"router_ignore_padding_tokens"},{anchor:"transformers.GPTSanJapaneseConfig.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>, default to <code>False</code>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.GPTSanJapaneseConfig.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to return the attentions tensors of all attention layers.`,name:"output_attentions"},{anchor:"transformers.GPTSanJapaneseConfig.initializer_factor",description:`<strong>initializer_factor</strong> (<code>float</code>, <em>optional</em>, defaults to 0.002) &#x2014;
A factor for initializing all weight matrices.`,name:"initializer_factor"},{anchor:"transformers.GPTSanJapaneseConfig.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>, default to <code>False</code>) &#x2014;
Whether or not to return the router logits of all experts.`,name:"output_router_logits"},{anchor:"transformers.GPTSanJapaneseConfig.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the model should return the last key/values attentions (not used by all models)`,name:"use_cache"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gptsan_japanese/configuration_gptsan_japanese.py#L29"}}),Ce=new Q({}),Pe=new Ue({props:{name:"class transformers.GPTSanJapaneseTokenizer",anchor:"transformers.GPTSanJapaneseTokenizer",parameters:[{name:"vocab_file",val:""},{name:"emoji_file",val:""},{name:"unk_token",val:" = '<|nottoken|>'"},{name:"pad_token",val:" = '<|separator|>'"},{name:"bos_token",val:" = '<|startoftext|>'"},{name:"eos_token",val:" = '<|endoftext|>'"},{name:"sep_token",val:" = '<|segmenter|>'"},{name:"do_clean_text",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTSanJapaneseTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
File containing the vocabulary.`,name:"vocab_file"},{anchor:"transformers.GPTSanJapaneseTokenizer.emoji_file",description:`<strong>emoji_file</strong> (<code>str</code>) &#x2014;
File containing the emoji.`,name:"emoji_file"},{anchor:"transformers.GPTSanJapaneseTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|nottoken|&gt;&quot;</code>) &#x2014;
The token used for unknown charactor`,name:"unk_token"},{anchor:"transformers.GPTSanJapaneseTokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|separator|&gt;&quot;</code>) &#x2014;
The token used for padding`,name:"pad_token"},{anchor:"transformers.GPTSanJapaneseTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|startoftext|&gt;&quot;&quot;</code>) &#x2014;
The beginning of sequence token.`,name:"bos_token"},{anchor:"transformers.GPTSanJapaneseTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|endoftext|&gt;&quot;</code>) &#x2014;
The end of sequence token.`,name:"eos_token"},{anchor:"transformers.GPTSanJapaneseTokenizer.sep_token",description:`<strong>sep_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|segmenter|&gt;&quot;</code>) &#x2014;
A special token to separate token to prefix part and general input part.`,name:"sep_token"},{anchor:"transformers.GPTSanJapaneseTokenizer.do_clean_text",description:`<strong>do_clean_text</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to clean text for URL, EMAIL, TEL, Japanese DATE and Japanese PRICE.`,name:"do_clean_text"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gptsan_japanese/tokenization_gptsan_japanese.py#L78"}}),ce=new nt({props:{anchor:"transformers.GPTSanJapaneseTokenizer.example",$$slots:{default:[fl]},$$scope:{ctx:G}}}),he=new nt({props:{anchor:"transformers.GPTSanJapaneseTokenizer.example-2",$$slots:{default:[gl]},$$scope:{ctx:G}}}),me=new nt({props:{anchor:"transformers.GPTSanJapaneseTokenizer.example-3",$$slots:{default:[_l]},$$scope:{ctx:G}}}),Re=new Ue({props:{name:"convert_tokens_to_string",anchor:"transformers.GPTSanJapaneseTokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gptsan_japanese/tokenization_gptsan_japanese.py#L222"}}),Ie=new Ue({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.GPTSanJapaneseTokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gptsan_japanese/tokenization_gptsan_japanese.py#L303"}}),fe=new nt({props:{anchor:"transformers.GPTSanJapaneseTokenizer.create_token_type_ids_from_sequences.example",$$slots:{default:[Tl]},$$scope:{ctx:G}}}),ze=new Q({}),Qe=new Ue({props:{name:"class transformers.GPTSanJapaneseModel",anchor:"transformers.GPTSanJapaneseModel",parameters:[{name:"config",val:": GPTSanJapaneseConfig"}],parametersDescription:[{anchor:"transformers.GPTSanJapaneseModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.31.0/en/model_doc/gptsan-japanese#transformers.GPTSanJapaneseConfig">GPTSanJapaneseConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/v4.31.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gptsan_japanese/modeling_gptsan_japanese.py#L868"}}),Oe=new Ue({props:{name:"forward",anchor:"transformers.GPTSanJapaneseModel.forward",parameters:[{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"token_type_ids",val:": typing.Optional[torch.FloatTensor] = None"},{name:"spout",val:": typing.Optional[torch.FloatTensor] = None"},{name:"past_key_values",val:": typing.Optional[typing.Tuple[typing.Tuple[torch.FloatTensor]]] = None"},{name:"head_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"use_cache",val:": typing.Optional[bool] = False"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"decoder_inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"},{name:"output_router_logits",val:": typing.Optional[bool] = None"},{name:"num_precontext",val:": typing.Optional[torch.LongTensor] = None"}],parametersDescription:[{anchor:"transformers.GPTSanJapaneseModel.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. GPTSAN-japanese is a model that generates sentence
continuations or predicts tokens at mask positions. Special tokens required for inputs to the model are
automatically appended.`,name:"input_ids"},{anchor:"transformers.GPTSanJapaneseModel.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.GPTSanJapaneseModel.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
An input that masks the Prefix part in the Prefix-LM input. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>prefix</strong> input,</li>
<li>0 for tokens that are <strong>not-prefix</strong> input.</li>
</ul>`,name:"token_type_ids"},{anchor:"transformers.GPTSanJapaneseModel.forward.spout",description:`<strong>spout</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, config.d_spout)</code>) &#x2014;
This vector is transformed through an 8-layer FFN and can be used instead of <code>past_key_values</code>.`,name:"spout"},{anchor:"transformers.GPTSanJapaneseModel.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code> of length <code>config.n_layers</code> with each tuple having 4 tensors of shape <code>(batch_size, num_heads, sequence_length - 1, embed_size_per_head)</code>) &#x2014;
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.GPTSanJapaneseModel.forward.head_mask",description:`<strong>head_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(num_heads,)</code> or <code>(num_layers, num_heads)</code>, <em>optional</em>) &#x2014;
Mask to nullify selected heads of the self-attention modules. Mask values selected in <code>[0, 1]</code>:`,name:"head_mask"},{anchor:"transformers.GPTSanJapaneseModel.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.GPTSanJapaneseModel.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.GPTSanJapaneseModel.forward.decoder_inputs_embeds",description:`<strong>decoder_inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, target_sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>decoder_input_ids</code> you can choose to directly pass an embedded
representation. If <code>past_key_values</code> is used, optionally only the last <code>decoder_inputs_embeds</code> have to be
input (see <code>past_key_values</code>). This is useful if you want more control over how to convert
<code>decoder_input_ids</code> indices into associated vectors than the model&#x2019;s internal embedding lookup matrix.`,name:"decoder_inputs_embeds"},{anchor:"transformers.GPTSanJapaneseModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.GPTSanJapaneseModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.GPTSanJapaneseModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.31.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.GPTSanJapaneseModel.forward.router_logits",description:`<strong>router_logits</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_router_logits=True</code> is passed or when <code>config.add_router_probs=True</code>) &#x2014;
Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, sequence_length, num_experts)</code>.
Router logits of the decoder model, useful to compute the auxiliary loss for Mixture of Experts models.`,name:"router_logits"},{anchor:"transformers.GPTSanJapaneseModel.forward.num_precontext",description:`<strong>num_precontext</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size,1)</code>) &#x2014;
length of <code>hybrid</code> input tokens in the input. Tokens up to this length refer to both front and back like
BERT, tokens after that refer only to front like GPT. see also:
<a href="https://github.com/tanreinama/GPTSAN/blob/main/report/model.md" rel="nofollow">https://github.com/tanreinama/GPTSAN/blob/main/report/model.md</a>`,name:"num_precontext"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gptsan_japanese/modeling_gptsan_japanese.py#L902",returnDescription:`
<p><code>MoEModelOutputWithPastAndCrossAttentions</code> or <code>tuple</code> if <code>return_dict</code> returns
MoEModelOutputWithPastAndCrossAttentions insted of tuple</p>
`}}),_e=new pl({props:{$$slots:{default:[yl]},$$scope:{ctx:G}}}),We=new Q({}),Xe=new Ue({props:{name:"class transformers.GPTSanJapaneseForConditionalGeneration",anchor:"transformers.GPTSanJapaneseForConditionalGeneration",parameters:[{name:"config",val:": GPTSanJapaneseConfig"}],parametersDescription:[{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.31.0/en/model_doc/gptsan-japanese#transformers.GPTSanJapaneseConfig">GPTSanJapaneseConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/v4.31.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gptsan_japanese/modeling_gptsan_japanese.py#L1113"}}),Ke=new Ue({props:{name:"forward",anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward",parameters:[{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"token_type_ids",val:": typing.Optional[torch.FloatTensor] = None"},{name:"spout",val:": typing.Optional[torch.FloatTensor] = None"},{name:"past_key_values",val:": typing.Optional[typing.Tuple[typing.Tuple[torch.FloatTensor]]] = None"},{name:"head_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"use_cache",val:": typing.Optional[bool] = False"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"decoder_inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"},{name:"output_router_logits",val:": typing.Optional[bool] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"}],parametersDescription:[{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. GPTSAN-japanese is a model that generates sentence
continuations or predicts tokens at mask positions. Special tokens required for inputs to the model are
automatically appended.`,name:"input_ids"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
An input that masks the Prefix part in the Prefix-LM input. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>prefix</strong> input,</li>
<li>0 for tokens that are <strong>not-prefix</strong> input.</li>
</ul>`,name:"token_type_ids"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.spout",description:`<strong>spout</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, config.d_spout)</code>) &#x2014;
This vector is transformed through an 8-layer FFN and can be used instead of <code>past_key_values</code>.`,name:"spout"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code> of length <code>config.n_layers</code> with each tuple having 4 tensors of shape <code>(batch_size, num_heads, sequence_length - 1, embed_size_per_head)</code>) &#x2014;
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.head_mask",description:`<strong>head_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(num_heads,)</code> or <code>(num_layers, num_heads)</code>, <em>optional</em>) &#x2014;
Mask to nullify selected heads of the self-attention modules. Mask values selected in <code>[0, 1]</code>:`,name:"head_mask"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.decoder_inputs_embeds",description:`<strong>decoder_inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, target_sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>decoder_input_ids</code> you can choose to directly pass an embedded
representation. If <code>past_key_values</code> is used, optionally only the last <code>decoder_inputs_embeds</code> have to be
input (see <code>past_key_values</code>). This is useful if you want more control over how to convert
<code>decoder_input_ids</code> indices into associated vectors than the model&#x2019;s internal embedding lookup matrix.`,name:"decoder_inputs_embeds"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.31.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.router_logits",description:`<strong>router_logits</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_router_logits=True</code> is passed or when <code>config.add_router_probs=True</code>) &#x2014;
Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, sequence_length, num_experts)</code>.
Router logits of the decoder model, useful to compute the auxiliary loss for Mixture of Experts models.`,name:"router_logits"},{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) &#x2014;
Labels for computing the sequence classification loss. Indices should be in <code>[-100, 0, ..., config.vocab_size - 1]</code>. All labels set to <code>-100</code> are ignored (masked), the loss is only computed for
labels in <code>[0, ..., config.vocab_size]</code>`,name:"labels"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/gptsan_japanese/modeling_gptsan_japanese.py#L1124",returnDescription:`
<p><code>MoECausalLMOutputWithPast</code> or <code>tuple</code> if <code>return_dict</code> returns MoECausalLMOutputWithPast insted of tuple</p>
`}}),ye=new pl({props:{$$slots:{default:[bl]},$$scope:{ctx:G}}}),be=new nt({props:{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.example",$$slots:{default:[Jl]},$$scope:{ctx:G}}}),Je=new nt({props:{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.example-2",$$slots:{default:[Ml]},$$scope:{ctx:G}}}),Me=new nt({props:{anchor:"transformers.GPTSanJapaneseForConditionalGeneration.forward.example-3",$$slots:{default:[Ul]},$$scope:{ctx:G}}}),{c(){p=a("meta"),T=c(),m=a("h1"),g=a("a"),j=a("span"),y(i.$$.fragment),_=c(),P=a("span"),ya=r("GPTSAN-japanese"),Vn=c(),A=a("h2"),ne=a("a"),vt=a("span"),y(je.$$.fragment),ba=c(),wt=a("span"),Ja=r("Overview"),xn=c(),st=a("p"),Ma=r("The GPTSAN-japanese model was released in the repository by Toshiyuki Sakamoto (tanreinama)."),$n=c(),ot=a("p"),Ua=r(`GPTSAN is a Japanese language model using Switch Transformer. It has the same structure as the model introduced as Prefix LM
in the T5 paper, and support both Text Generation and Masked Language Modeling tasks. These basic tasks similarly can
fine-tune for translation or summarization.`),En=c(),q=a("h3"),ae=a("a"),St=a("span"),y(ve.$$.fragment),ka=c(),Gt=a("span"),ja=r("Generation"),Cn=c(),se=a("p"),va=r("The "),Vt=a("code"),wa=r("generate()"),Sa=r(" method can be used to generate text using GPTSAN-Japanese model."),Pn=c(),y(we.$$.fragment),Nn=c(),Z=a("h2"),oe=a("a"),xt=a("span"),y(Se.$$.fragment),Ga=c(),$t=a("span"),Va=r("GPTSAN Features"),Fn=c(),rt=a("p"),xa=r(`GPTSAN has some unique features. It has a model structure of Prefix-LM. It works as a shifted Masked Language Model for Prefix Input tokens. Un-prefixed inputs behave like normal generative models.
The Spout vector is a GPTSAN specific input. Spout is pre-trained with random inputs, but you can specify a class of text or an arbitrary vector during fine-tuning. This allows you to indicate the tendency of the generated text.
GPTSAN has a sparse Feed Forward based on Switch-Transformer. You can also add other layers and train them partially. See the original GPTSAN repository for details.`),Rn=c(),B=a("h3"),re=a("a"),Et=a("span"),y(Ge.$$.fragment),$a=c(),Ct=a("span"),Ea=r("Prefix-LM Model"),In=c(),w=a("p"),Ca=r("GPTSAN has the structure of the model named Prefix-LM in the "),Pt=a("code"),Pa=r("T5"),Na=r(" paper. (The original GPTSAN repository calls it "),Nt=a("code"),Fa=r("hybrid"),Ra=r(`)
In GPTSAN, the `),Ft=a("code"),Ia=r("Prefix"),za=r(` part of Prefix-LM, that is, the input position that can be referenced by both tokens, can be specified with any length.
Arbitrary lengths can also be specified differently for each batch.
This length applies to the text entered in `),Rt=a("code"),Qa=r("prefix_text"),Aa=r(` for the tokenizer.
The tokenizer returns the mask of the `),It=a("code"),qa=r("Prefix"),Za=r(" part of Prefix-LM as "),zt=a("code"),Ba=r("token_type_ids"),Oa=r(`.
The model treats the part where `),Qt=a("code"),Wa=r("token_type_ids"),Xa=r(" is 1 as a "),At=a("code"),Da=r("Prefix"),Ya=r(" part, that is, the input can refer to both tokens before and after."),zn=c(),lt=a("p"),La=r("Tips:"),Qn=c(),it=a("p"),Ha=r(`Specifying the Prefix part is done with a mask passed to self-attention.
When token_type_ids=None or all zero, it is equivalent to regular causal mask`),An=c(),pt=a("p"),Ka=r("for example:"),qn=c(),dt=a("blockquote"),qt=a("blockquote"),Zt=a("blockquote"),Bt=a("p"),es=r(`x_token = tokenizer(\u201C\uFF71\uFF72\uFF73\uFF74\u201D)
input_ids:      | SOT | SEG | \uFF71 | \uFF72 | \uFF73 | \uFF74 |
token_type_ids: | 1   | 0   | 0 | 0 | 0 | 0 |
prefix_lm_mask:
SOT | 1 0 0 0 0 0 |
SEG | 1 1 0 0 0 0 |
\uFF71   | 1 1 1 0 0 0 |
\uFF72   | 1 1 1 1 0 0 |
\uFF73   | 1 1 1 1 1 0 |
\uFF74   | 1 1 1 1 1 1 |`),Zn=c(),ct=a("blockquote"),Ot=a("blockquote"),Wt=a("blockquote"),Xt=a("p"),ts=r(`x_token = tokenizer("", prefix_text=\u201C\uFF71\uFF72\uFF73\uFF74\u201D)
input_ids:      | SOT | \uFF71 | \uFF72 | \uFF73 | \uFF74 | SEG |
token_type_ids: | 1   | 1 | 1 | 1 | 1 | 0  |
prefix_lm_mask:
SOT | 1 1 1 1 1 0 |
\uFF71   | 1 1 1 1 1 0 |
\uFF72   | 1 1 1 1 1 0 |
\uFF73   | 1 1 1 1 1 0 |
\uFF74   | 1 1 1 1 1 0 |
SEG | 1 1 1 1 1 1 |`),Bn=c(),ht=a("blockquote"),Dt=a("blockquote"),Yt=a("blockquote"),Lt=a("p"),ns=r(`x_token = tokenizer(\u201C\uFF73\uFF74\u201D, prefix_text=\u201C\uFF71\uFF72\u201D)
input_ids:      | SOT | \uFF71 | \uFF72 | SEG | \uFF73 | \uFF74 |
token_type_ids: | 1   | 1 | 1 | 0   | 0 | 0 |
prefix_lm_mask:
SOT | 1 1 1 0 0 0 |
\uFF71   | 1 1 1 0 0 0 |
\uFF72   | 1 1 1 0 0 0 |
SEG | 1 1 1 1 0 0 |
\uFF73   | 1 1 1 1 1 0 |
\uFF74   | 1 1 1 1 1 1 |`),On=c(),O=a("h3"),le=a("a"),Ht=a("span"),y(Ve.$$.fragment),as=c(),Kt=a("span"),ss=r("Spout Vector"),Wn=c(),ie=a("p"),os=r(`A Spout Vector is a special vector for controlling text generation.
This vector is treated as the first embedding in self-attention to bring extraneous attention to the generated tokens.
In the pre-trained model published from `),en=a("code"),rs=r("Tanrei/GPTSAN-japanese"),ls=r(`, the Spout Vector is a 128-dimensional vector that passes through 8 fully connected layers in the model and is projected into the space acting as external attention.
The Spout Vector projected by the fully connected layer is split to be passed to all self-attentions.`),Xn=c(),W=a("h2"),pe=a("a"),tn=a("span"),y(xe.$$.fragment),is=c(),nn=a("span"),ps=r("GPTSanJapaneseConfig"),Dn=c(),N=a("div"),y($e.$$.fragment),ds=c(),X=a("p"),cs=r("This is the configuration class to store the configuration of a "),mt=a("a"),hs=r("GPTSanJapaneseModel"),ms=r(`. It is used to instantiate
a GPTSANJapanese model according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of the GPTSANJapanese
`),Ee=a("a"),us=r("Tanrei/GPTSAN-japanese"),fs=r(" architecture."),gs=c(),D=a("p"),_s=r("Configuration objects inherit from "),ut=a("a"),Ts=r("PretrainedConfig"),ys=r(` and can be used to control the model outputs. Read the
documentation from `),ft=a("a"),bs=r("PretrainedConfig"),Js=r(" for more information."),Yn=c(),Y=a("h2"),de=a("a"),an=a("span"),y(Ce.$$.fragment),Ms=c(),sn=a("span"),Us=r("GPTSanJapaneseTokenizer"),Ln=c(),S=a("div"),y(Pe.$$.fragment),ks=c(),on=a("p"),js=r("This tokenizer is based on GPTNeoXJapaneseTokenizer and has the following modifications"),vs=c(),L=a("ul"),rn=a("li"),ws=r("Decoding byte0~byte255 tokens correctly"),Ss=c(),ln=a("li"),Gs=r("Added bagofword token handling"),Vs=c(),Ne=a("li"),xs=r(`Return token_type_ids for Prefix-LM model
The bagofword token represents a repetition of the previous token and is converted to 3 consecutive tokens when
decoding In addition, the original Japanese special Sub-Word-Encoding has been released in this repository
(`),Fe=a("a"),$s=r("https://github.com/tanreinama/Japanese-BPEEncoder_V2"),Es=r(`). The token_type_ids is a mask indicating the prefix input
position of the Prefix-LM model. To specify a prefix position, specify a prefix input for prefix_text, or specify a
sentence of the prefix part and the part after it as a text pair of batch input.`),Cs=c(),y(ce.$$.fragment),Ps=c(),y(he.$$.fragment),Ns=c(),y(me.$$.fragment),Fs=c(),ue=a("div"),y(Re.$$.fragment),Rs=c(),pn=a("p"),Is=r("Converts a sequence of tokens (string) in a single string."),zs=c(),F=a("div"),y(Ie.$$.fragment),Qs=c(),dn=a("p"),As=r(`The tokenizer returns token_type_ids as separators between the Prefix part and the rest.
token_type_ids is 1 for the Prefix part and 0 for the rest of the token.`),qs=c(),y(fe.$$.fragment),Hn=c(),H=a("h2"),ge=a("a"),cn=a("span"),y(ze.$$.fragment),Zs=c(),hn=a("span"),Bs=r("GPTSanJapaneseModel"),Kn=c(),E=a("div"),y(Qe.$$.fragment),Os=c(),mn=a("p"),Ws=r("The bare GPTSAN-japanese Model transformer outputting raw hidden-states without any specific head on top."),Xs=c(),Ae=a("p"),Ds=r("The "),qe=a("a"),Ys=r("GPTSAN-japanese"),Ls=r(` model was proposed in General-purpose Swich transformer
based Japanese language model`),Hs=c(),Ze=a("p"),Ks=r("This model is also a PyTorch "),Be=a("a"),eo=r("torch.nn.Module"),to=r(` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),no=c(),R=a("div"),y(Oe.$$.fragment),ao=c(),K=a("p"),so=r("The "),gt=a("a"),oo=r("GPTSanJapaneseModel"),ro=r(" forward method, overrides the "),un=a("code"),lo=r("__call__"),io=r(" special method."),po=c(),y(_e.$$.fragment),ea=c(),ee=a("h2"),Te=a("a"),fn=a("span"),y(We.$$.fragment),co=c(),gn=a("span"),ho=r("GPTSanJapaneseForConditionalGeneration"),ta=c(),C=a("div"),y(Xe.$$.fragment),mo=c(),_n=a("p"),uo=r("The bare GPTSAN-japanese Model with a language modeling head."),fo=c(),De=a("p"),go=r("The "),Ye=a("a"),_o=r("GPTSAN-japanese"),To=r(` model was proposed in General-purpose Swich transformer
based Japanese language model`),yo=c(),Le=a("p"),bo=r("This model is also a PyTorch "),He=a("a"),Jo=r("torch.nn.Module"),Mo=r(` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),Uo=c(),v=a("div"),y(Ke.$$.fragment),ko=c(),te=a("p"),jo=r("The "),_t=a("a"),vo=r("GPTSanJapaneseForConditionalGeneration"),wo=r(" forward method, overrides the "),Tn=a("code"),So=r("__call__"),Go=r(" special method."),Vo=c(),y(ye.$$.fragment),xo=c(),yn=a("p"),$o=r("Example:"),Eo=c(),bn=a("p"),Co=r("Text Generation with regular LM Model"),Po=c(),y(be.$$.fragment),No=c(),Jn=a("p"),Fo=r("Text Generation with Prefix-LM Model"),Ro=c(),y(Je.$$.fragment),Io=c(),Mn=a("p"),zo=r("Simultaneously Text Generation And Masked Language Model"),Qo=c(),y(Me.$$.fragment),this.h()},l(e){const d=ml('[data-svelte="svelte-1phssyn"]',document.head);p=s(d,"META",{name:!0,content:!0}),d.forEach(n),T=h(e),m=s(e,"H1",{class:!0});var et=o(m);g=s(et,"A",{id:!0,class:!0,href:!0});var Un=o(g);j=s(Un,"SPAN",{});var kn=o(j);b(i.$$.fragment,kn),kn.forEach(n),Un.forEach(n),_=h(et),P=s(et,"SPAN",{});var jn=o(P);ya=l(jn,"GPTSAN-japanese"),jn.forEach(n),et.forEach(n),Vn=h(e),A=s(e,"H2",{class:!0});var tt=o(A);ne=s(tt,"A",{id:!0,class:!0,href:!0});var vn=o(ne);vt=s(vn,"SPAN",{});var wn=o(vt);b(je.$$.fragment,wn),wn.forEach(n),vn.forEach(n),ba=h(tt),wt=s(tt,"SPAN",{});var Sn=o(wt);Ja=l(Sn,"Overview"),Sn.forEach(n),tt.forEach(n),xn=h(e),st=s(e,"P",{});var Gn=o(st);Ma=l(Gn,"The GPTSAN-japanese model was released in the repository by Toshiyuki Sakamoto (tanreinama)."),Gn.forEach(n),$n=h(e),ot=s(e,"P",{});var Ao=o(ot);Ua=l(Ao,`GPTSAN is a Japanese language model using Switch Transformer. It has the same structure as the model introduced as Prefix LM
in the T5 paper, and support both Text Generation and Masked Language Modeling tasks. These basic tasks similarly can
fine-tune for translation or summarization.`),Ao.forEach(n),En=h(e),q=s(e,"H3",{class:!0});var aa=o(q);ae=s(aa,"A",{id:!0,class:!0,href:!0});var qo=o(ae);St=s(qo,"SPAN",{});var Zo=o(St);b(ve.$$.fragment,Zo),Zo.forEach(n),qo.forEach(n),ka=h(aa),Gt=s(aa,"SPAN",{});var Bo=o(Gt);ja=l(Bo,"Generation"),Bo.forEach(n),aa.forEach(n),Cn=h(e),se=s(e,"P",{});var sa=o(se);va=l(sa,"The "),Vt=s(sa,"CODE",{});var Oo=o(Vt);wa=l(Oo,"generate()"),Oo.forEach(n),Sa=l(sa," method can be used to generate text using GPTSAN-Japanese model."),sa.forEach(n),Pn=h(e),b(we.$$.fragment,e),Nn=h(e),Z=s(e,"H2",{class:!0});var oa=o(Z);oe=s(oa,"A",{id:!0,class:!0,href:!0});var Wo=o(oe);xt=s(Wo,"SPAN",{});var Xo=o(xt);b(Se.$$.fragment,Xo),Xo.forEach(n),Wo.forEach(n),Ga=h(oa),$t=s(oa,"SPAN",{});var Do=o($t);Va=l(Do,"GPTSAN Features"),Do.forEach(n),oa.forEach(n),Fn=h(e),rt=s(e,"P",{});var Yo=o(rt);xa=l(Yo,`GPTSAN has some unique features. It has a model structure of Prefix-LM. It works as a shifted Masked Language Model for Prefix Input tokens. Un-prefixed inputs behave like normal generative models.
The Spout vector is a GPTSAN specific input. Spout is pre-trained with random inputs, but you can specify a class of text or an arbitrary vector during fine-tuning. This allows you to indicate the tendency of the generated text.
GPTSAN has a sparse Feed Forward based on Switch-Transformer. You can also add other layers and train them partially. See the original GPTSAN repository for details.`),Yo.forEach(n),Rn=h(e),B=s(e,"H3",{class:!0});var ra=o(B);re=s(ra,"A",{id:!0,class:!0,href:!0});var Lo=o(re);Et=s(Lo,"SPAN",{});var Ho=o(Et);b(Ge.$$.fragment,Ho),Ho.forEach(n),Lo.forEach(n),$a=h(ra),Ct=s(ra,"SPAN",{});var Ko=o(Ct);Ea=l(Ko,"Prefix-LM Model"),Ko.forEach(n),ra.forEach(n),In=h(e),w=s(e,"P",{});var x=o(w);Ca=l(x,"GPTSAN has the structure of the model named Prefix-LM in the "),Pt=s(x,"CODE",{});var er=o(Pt);Pa=l(er,"T5"),er.forEach(n),Na=l(x," paper. (The original GPTSAN repository calls it "),Nt=s(x,"CODE",{});var tr=o(Nt);Fa=l(tr,"hybrid"),tr.forEach(n),Ra=l(x,`)
In GPTSAN, the `),Ft=s(x,"CODE",{});var nr=o(Ft);Ia=l(nr,"Prefix"),nr.forEach(n),za=l(x,` part of Prefix-LM, that is, the input position that can be referenced by both tokens, can be specified with any length.
Arbitrary lengths can also be specified differently for each batch.
This length applies to the text entered in `),Rt=s(x,"CODE",{});var ar=o(Rt);Qa=l(ar,"prefix_text"),ar.forEach(n),Aa=l(x,` for the tokenizer.
The tokenizer returns the mask of the `),It=s(x,"CODE",{});var sr=o(It);qa=l(sr,"Prefix"),sr.forEach(n),Za=l(x," part of Prefix-LM as "),zt=s(x,"CODE",{});var or=o(zt);Ba=l(or,"token_type_ids"),or.forEach(n),Oa=l(x,`.
The model treats the part where `),Qt=s(x,"CODE",{});var rr=o(Qt);Wa=l(rr,"token_type_ids"),rr.forEach(n),Xa=l(x," is 1 as a "),At=s(x,"CODE",{});var lr=o(At);Da=l(lr,"Prefix"),lr.forEach(n),Ya=l(x," part, that is, the input can refer to both tokens before and after."),x.forEach(n),zn=h(e),lt=s(e,"P",{});var ir=o(lt);La=l(ir,"Tips:"),ir.forEach(n),Qn=h(e),it=s(e,"P",{});var pr=o(it);Ha=l(pr,`Specifying the Prefix part is done with a mask passed to self-attention.
When token_type_ids=None or all zero, it is equivalent to regular causal mask`),pr.forEach(n),An=h(e),pt=s(e,"P",{});var dr=o(pt);Ka=l(dr,"for example:"),dr.forEach(n),qn=h(e),dt=s(e,"BLOCKQUOTE",{});var cr=o(dt);qt=s(cr,"BLOCKQUOTE",{});var hr=o(qt);Zt=s(hr,"BLOCKQUOTE",{});var mr=o(Zt);Bt=s(mr,"P",{});var ur=o(Bt);es=l(ur,`x_token = tokenizer(\u201C\uFF71\uFF72\uFF73\uFF74\u201D)
input_ids:      | SOT | SEG | \uFF71 | \uFF72 | \uFF73 | \uFF74 |
token_type_ids: | 1   | 0   | 0 | 0 | 0 | 0 |
prefix_lm_mask:
SOT | 1 0 0 0 0 0 |
SEG | 1 1 0 0 0 0 |
\uFF71   | 1 1 1 0 0 0 |
\uFF72   | 1 1 1 1 0 0 |
\uFF73   | 1 1 1 1 1 0 |
\uFF74   | 1 1 1 1 1 1 |`),ur.forEach(n),mr.forEach(n),hr.forEach(n),cr.forEach(n),Zn=h(e),ct=s(e,"BLOCKQUOTE",{});var fr=o(ct);Ot=s(fr,"BLOCKQUOTE",{});var gr=o(Ot);Wt=s(gr,"BLOCKQUOTE",{});var _r=o(Wt);Xt=s(_r,"P",{});var Tr=o(Xt);ts=l(Tr,`x_token = tokenizer("", prefix_text=\u201C\uFF71\uFF72\uFF73\uFF74\u201D)
input_ids:      | SOT | \uFF71 | \uFF72 | \uFF73 | \uFF74 | SEG |
token_type_ids: | 1   | 1 | 1 | 1 | 1 | 0  |
prefix_lm_mask:
SOT | 1 1 1 1 1 0 |
\uFF71   | 1 1 1 1 1 0 |
\uFF72   | 1 1 1 1 1 0 |
\uFF73   | 1 1 1 1 1 0 |
\uFF74   | 1 1 1 1 1 0 |
SEG | 1 1 1 1 1 1 |`),Tr.forEach(n),_r.forEach(n),gr.forEach(n),fr.forEach(n),Bn=h(e),ht=s(e,"BLOCKQUOTE",{});var yr=o(ht);Dt=s(yr,"BLOCKQUOTE",{});var br=o(Dt);Yt=s(br,"BLOCKQUOTE",{});var Jr=o(Yt);Lt=s(Jr,"P",{});var Mr=o(Lt);ns=l(Mr,`x_token = tokenizer(\u201C\uFF73\uFF74\u201D, prefix_text=\u201C\uFF71\uFF72\u201D)
input_ids:      | SOT | \uFF71 | \uFF72 | SEG | \uFF73 | \uFF74 |
token_type_ids: | 1   | 1 | 1 | 0   | 0 | 0 |
prefix_lm_mask:
SOT | 1 1 1 0 0 0 |
\uFF71   | 1 1 1 0 0 0 |
\uFF72   | 1 1 1 0 0 0 |
SEG | 1 1 1 1 0 0 |
\uFF73   | 1 1 1 1 1 0 |
\uFF74   | 1 1 1 1 1 1 |`),Mr.forEach(n),Jr.forEach(n),br.forEach(n),yr.forEach(n),On=h(e),O=s(e,"H3",{class:!0});var la=o(O);le=s(la,"A",{id:!0,class:!0,href:!0});var Ur=o(le);Ht=s(Ur,"SPAN",{});var kr=o(Ht);b(Ve.$$.fragment,kr),kr.forEach(n),Ur.forEach(n),as=h(la),Kt=s(la,"SPAN",{});var jr=o(Kt);ss=l(jr,"Spout Vector"),jr.forEach(n),la.forEach(n),Wn=h(e),ie=s(e,"P",{});var ia=o(ie);os=l(ia,`A Spout Vector is a special vector for controlling text generation.
This vector is treated as the first embedding in self-attention to bring extraneous attention to the generated tokens.
In the pre-trained model published from `),en=s(ia,"CODE",{});var vr=o(en);rs=l(vr,"Tanrei/GPTSAN-japanese"),vr.forEach(n),ls=l(ia,`, the Spout Vector is a 128-dimensional vector that passes through 8 fully connected layers in the model and is projected into the space acting as external attention.
The Spout Vector projected by the fully connected layer is split to be passed to all self-attentions.`),ia.forEach(n),Xn=h(e),W=s(e,"H2",{class:!0});var pa=o(W);pe=s(pa,"A",{id:!0,class:!0,href:!0});var wr=o(pe);tn=s(wr,"SPAN",{});var Sr=o(tn);b(xe.$$.fragment,Sr),Sr.forEach(n),wr.forEach(n),is=h(pa),nn=s(pa,"SPAN",{});var Gr=o(nn);ps=l(Gr,"GPTSanJapaneseConfig"),Gr.forEach(n),pa.forEach(n),Dn=h(e),N=s(e,"DIV",{class:!0});var Tt=o(N);b($e.$$.fragment,Tt),ds=h(Tt),X=s(Tt,"P",{});var yt=o(X);cs=l(yt,"This is the configuration class to store the configuration of a "),mt=s(yt,"A",{href:!0});var Vr=o(mt);hs=l(Vr,"GPTSanJapaneseModel"),Vr.forEach(n),ms=l(yt,`. It is used to instantiate
a GPTSANJapanese model according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of the GPTSANJapanese
`),Ee=s(yt,"A",{href:!0,rel:!0});var xr=o(Ee);us=l(xr,"Tanrei/GPTSAN-japanese"),xr.forEach(n),fs=l(yt," architecture."),yt.forEach(n),gs=h(Tt),D=s(Tt,"P",{});var bt=o(D);_s=l(bt,"Configuration objects inherit from "),ut=s(bt,"A",{href:!0});var $r=o(ut);Ts=l($r,"PretrainedConfig"),$r.forEach(n),ys=l(bt,` and can be used to control the model outputs. Read the
documentation from `),ft=s(bt,"A",{href:!0});var Er=o(ft);bs=l(Er,"PretrainedConfig"),Er.forEach(n),Js=l(bt," for more information."),bt.forEach(n),Tt.forEach(n),Yn=h(e),Y=s(e,"H2",{class:!0});var da=o(Y);de=s(da,"A",{id:!0,class:!0,href:!0});var Cr=o(de);an=s(Cr,"SPAN",{});var Pr=o(an);b(Ce.$$.fragment,Pr),Pr.forEach(n),Cr.forEach(n),Ms=h(da),sn=s(da,"SPAN",{});var Nr=o(sn);Us=l(Nr,"GPTSanJapaneseTokenizer"),Nr.forEach(n),da.forEach(n),Ln=h(e),S=s(e,"DIV",{class:!0});var $=o(S);b(Pe.$$.fragment,$),ks=h($),on=s($,"P",{});var Fr=o(on);js=l(Fr,"This tokenizer is based on GPTNeoXJapaneseTokenizer and has the following modifications"),Fr.forEach(n),vs=h($),L=s($,"UL",{});var Jt=o(L);rn=s(Jt,"LI",{});var Rr=o(rn);ws=l(Rr,"Decoding byte0~byte255 tokens correctly"),Rr.forEach(n),Ss=h(Jt),ln=s(Jt,"LI",{});var Ir=o(ln);Gs=l(Ir,"Added bagofword token handling"),Ir.forEach(n),Vs=h(Jt),Ne=s(Jt,"LI",{});var ca=o(Ne);xs=l(ca,`Return token_type_ids for Prefix-LM model
The bagofword token represents a repetition of the previous token and is converted to 3 consecutive tokens when
decoding In addition, the original Japanese special Sub-Word-Encoding has been released in this repository
(`),Fe=s(ca,"A",{href:!0,rel:!0});var zr=o(Fe);$s=l(zr,"https://github.com/tanreinama/Japanese-BPEEncoder_V2"),zr.forEach(n),Es=l(ca,`). The token_type_ids is a mask indicating the prefix input
position of the Prefix-LM model. To specify a prefix position, specify a prefix input for prefix_text, or specify a
sentence of the prefix part and the part after it as a text pair of batch input.`),ca.forEach(n),Jt.forEach(n),Cs=h($),b(ce.$$.fragment,$),Ps=h($),b(he.$$.fragment,$),Ns=h($),b(me.$$.fragment,$),Fs=h($),ue=s($,"DIV",{class:!0});var ha=o(ue);b(Re.$$.fragment,ha),Rs=h(ha),pn=s(ha,"P",{});var Qr=o(pn);Is=l(Qr,"Converts a sequence of tokens (string) in a single string."),Qr.forEach(n),ha.forEach(n),zs=h($),F=s($,"DIV",{class:!0});var Mt=o(F);b(Ie.$$.fragment,Mt),Qs=h(Mt),dn=s(Mt,"P",{});var Ar=o(dn);As=l(Ar,`The tokenizer returns token_type_ids as separators between the Prefix part and the rest.
token_type_ids is 1 for the Prefix part and 0 for the rest of the token.`),Ar.forEach(n),qs=h(Mt),b(fe.$$.fragment,Mt),Mt.forEach(n),$.forEach(n),Hn=h(e),H=s(e,"H2",{class:!0});var ma=o(H);ge=s(ma,"A",{id:!0,class:!0,href:!0});var qr=o(ge);cn=s(qr,"SPAN",{});var Zr=o(cn);b(ze.$$.fragment,Zr),Zr.forEach(n),qr.forEach(n),Zs=h(ma),hn=s(ma,"SPAN",{});var Br=o(hn);Bs=l(Br,"GPTSanJapaneseModel"),Br.forEach(n),ma.forEach(n),Kn=h(e),E=s(e,"DIV",{class:!0});var I=o(E);b(Qe.$$.fragment,I),Os=h(I),mn=s(I,"P",{});var Or=o(mn);Ws=l(Or,"The bare GPTSAN-japanese Model transformer outputting raw hidden-states without any specific head on top."),Or.forEach(n),Xs=h(I),Ae=s(I,"P",{});var ua=o(Ae);Ds=l(ua,"The "),qe=s(ua,"A",{href:!0,rel:!0});var Wr=o(qe);Ys=l(Wr,"GPTSAN-japanese"),Wr.forEach(n),Ls=l(ua,` model was proposed in General-purpose Swich transformer
based Japanese language model`),ua.forEach(n),Hs=h(I),Ze=s(I,"P",{});var fa=o(Ze);Ks=l(fa,"This model is also a PyTorch "),Be=s(fa,"A",{href:!0,rel:!0});var Xr=o(Be);eo=l(Xr,"torch.nn.Module"),Xr.forEach(n),to=l(fa,` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),fa.forEach(n),no=h(I),R=s(I,"DIV",{class:!0});var Ut=o(R);b(Oe.$$.fragment,Ut),ao=h(Ut),K=s(Ut,"P",{});var kt=o(K);so=l(kt,"The "),gt=s(kt,"A",{href:!0});var Dr=o(gt);oo=l(Dr,"GPTSanJapaneseModel"),Dr.forEach(n),ro=l(kt," forward method, overrides the "),un=s(kt,"CODE",{});var Yr=o(un);lo=l(Yr,"__call__"),Yr.forEach(n),io=l(kt," special method."),kt.forEach(n),po=h(Ut),b(_e.$$.fragment,Ut),Ut.forEach(n),I.forEach(n),ea=h(e),ee=s(e,"H2",{class:!0});var ga=o(ee);Te=s(ga,"A",{id:!0,class:!0,href:!0});var Lr=o(Te);fn=s(Lr,"SPAN",{});var Hr=o(fn);b(We.$$.fragment,Hr),Hr.forEach(n),Lr.forEach(n),co=h(ga),gn=s(ga,"SPAN",{});var Kr=o(gn);ho=l(Kr,"GPTSanJapaneseForConditionalGeneration"),Kr.forEach(n),ga.forEach(n),ta=h(e),C=s(e,"DIV",{class:!0});var z=o(C);b(Xe.$$.fragment,z),mo=h(z),_n=s(z,"P",{});var el=o(_n);uo=l(el,"The bare GPTSAN-japanese Model with a language modeling head."),el.forEach(n),fo=h(z),De=s(z,"P",{});var _a=o(De);go=l(_a,"The "),Ye=s(_a,"A",{href:!0,rel:!0});var tl=o(Ye);_o=l(tl,"GPTSAN-japanese"),tl.forEach(n),To=l(_a,` model was proposed in General-purpose Swich transformer
based Japanese language model`),_a.forEach(n),yo=h(z),Le=s(z,"P",{});var Ta=o(Le);bo=l(Ta,"This model is also a PyTorch "),He=s(Ta,"A",{href:!0,rel:!0});var nl=o(He);Jo=l(nl,"torch.nn.Module"),nl.forEach(n),Mo=l(Ta,` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),Ta.forEach(n),Uo=h(z),v=s(z,"DIV",{class:!0});var V=o(v);b(Ke.$$.fragment,V),ko=h(V),te=s(V,"P",{});var jt=o(te);jo=l(jt,"The "),_t=s(jt,"A",{href:!0});var al=o(_t);vo=l(al,"GPTSanJapaneseForConditionalGeneration"),al.forEach(n),wo=l(jt," forward method, overrides the "),Tn=s(jt,"CODE",{});var sl=o(Tn);So=l(sl,"__call__"),sl.forEach(n),Go=l(jt," special method."),jt.forEach(n),Vo=h(V),b(ye.$$.fragment,V),xo=h(V),yn=s(V,"P",{});var ol=o(yn);$o=l(ol,"Example:"),ol.forEach(n),Eo=h(V),bn=s(V,"P",{});var rl=o(bn);Co=l(rl,"Text Generation with regular LM Model"),rl.forEach(n),Po=h(V),b(be.$$.fragment,V),No=h(V),Jn=s(V,"P",{});var ll=o(Jn);Fo=l(ll,"Text Generation with Prefix-LM Model"),ll.forEach(n),Ro=h(V),b(Je.$$.fragment,V),Io=h(V),Mn=s(V,"P",{});var il=o(Mn);zo=l(il,"Simultaneously Text Generation And Masked Language Model"),il.forEach(n),Qo=h(V),b(Me.$$.fragment,V),V.forEach(n),z.forEach(n),this.h()},h(){u(p,"name","hf:doc:metadata"),u(p,"content",JSON.stringify(jl)),u(g,"id","gptsanjapanese"),u(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(g,"href","#gptsanjapanese"),u(m,"class","relative group"),u(ne,"id","overview"),u(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ne,"href","#overview"),u(A,"class","relative group"),u(ae,"id","generation"),u(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ae,"href","#generation"),u(q,"class","relative group"),u(oe,"id","gptsan-features"),u(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(oe,"href","#gptsan-features"),u(Z,"class","relative group"),u(re,"id","prefixlm-model"),u(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(re,"href","#prefixlm-model"),u(B,"class","relative group"),u(le,"id","spout-vector"),u(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(le,"href","#spout-vector"),u(O,"class","relative group"),u(pe,"id","transformers.GPTSanJapaneseConfig"),u(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(pe,"href","#transformers.GPTSanJapaneseConfig"),u(W,"class","relative group"),u(mt,"href","/docs/transformers/v4.31.0/en/model_doc/gptsan-japanese#transformers.GPTSanJapaneseModel"),u(Ee,"href","https://huggingface.co/Tanrei/GPTSAN-japanese"),u(Ee,"rel","nofollow"),u(ut,"href","/docs/transformers/v4.31.0/en/main_classes/configuration#transformers.PretrainedConfig"),u(ft,"href","/docs/transformers/v4.31.0/en/main_classes/configuration#transformers.PretrainedConfig"),u(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(de,"id","transformers.GPTSanJapaneseTokenizer"),u(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(de,"href","#transformers.GPTSanJapaneseTokenizer"),u(Y,"class","relative group"),u(Fe,"href","https://github.com/tanreinama/Japanese-BPEEncoder_V2"),u(Fe,"rel","nofollow"),u(ue,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(ge,"id","transformers.GPTSanJapaneseModel"),u(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ge,"href","#transformers.GPTSanJapaneseModel"),u(H,"class","relative group"),u(qe,"href","https://github.com/tanreinama/GPTSAN"),u(qe,"rel","nofollow"),u(Be,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),u(Be,"rel","nofollow"),u(gt,"href","/docs/transformers/v4.31.0/en/model_doc/gptsan-japanese#transformers.GPTSanJapaneseModel"),u(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(Te,"id","transformers.GPTSanJapaneseForConditionalGeneration"),u(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Te,"href","#transformers.GPTSanJapaneseForConditionalGeneration"),u(ee,"class","relative group"),u(Ye,"href","https://github.com/tanreinama/GPTSAN"),u(Ye,"rel","nofollow"),u(He,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),u(He,"rel","nofollow"),u(_t,"href","/docs/transformers/v4.31.0/en/model_doc/gptsan-japanese#transformers.GPTSanJapaneseForConditionalGeneration"),u(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),u(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,d){t(document.head,p),f(e,T,d),f(e,m,d),t(m,g),t(g,j),J(i,j,null),t(m,_),t(m,P),t(P,ya),f(e,Vn,d),f(e,A,d),t(A,ne),t(ne,vt),J(je,vt,null),t(A,ba),t(A,wt),t(wt,Ja),f(e,xn,d),f(e,st,d),t(st,Ma),f(e,$n,d),f(e,ot,d),t(ot,Ua),f(e,En,d),f(e,q,d),t(q,ae),t(ae,St),J(ve,St,null),t(q,ka),t(q,Gt),t(Gt,ja),f(e,Cn,d),f(e,se,d),t(se,va),t(se,Vt),t(Vt,wa),t(se,Sa),f(e,Pn,d),J(we,e,d),f(e,Nn,d),f(e,Z,d),t(Z,oe),t(oe,xt),J(Se,xt,null),t(Z,Ga),t(Z,$t),t($t,Va),f(e,Fn,d),f(e,rt,d),t(rt,xa),f(e,Rn,d),f(e,B,d),t(B,re),t(re,Et),J(Ge,Et,null),t(B,$a),t(B,Ct),t(Ct,Ea),f(e,In,d),f(e,w,d),t(w,Ca),t(w,Pt),t(Pt,Pa),t(w,Na),t(w,Nt),t(Nt,Fa),t(w,Ra),t(w,Ft),t(Ft,Ia),t(w,za),t(w,Rt),t(Rt,Qa),t(w,Aa),t(w,It),t(It,qa),t(w,Za),t(w,zt),t(zt,Ba),t(w,Oa),t(w,Qt),t(Qt,Wa),t(w,Xa),t(w,At),t(At,Da),t(w,Ya),f(e,zn,d),f(e,lt,d),t(lt,La),f(e,Qn,d),f(e,it,d),t(it,Ha),f(e,An,d),f(e,pt,d),t(pt,Ka),f(e,qn,d),f(e,dt,d),t(dt,qt),t(qt,Zt),t(Zt,Bt),t(Bt,es),f(e,Zn,d),f(e,ct,d),t(ct,Ot),t(Ot,Wt),t(Wt,Xt),t(Xt,ts),f(e,Bn,d),f(e,ht,d),t(ht,Dt),t(Dt,Yt),t(Yt,Lt),t(Lt,ns),f(e,On,d),f(e,O,d),t(O,le),t(le,Ht),J(Ve,Ht,null),t(O,as),t(O,Kt),t(Kt,ss),f(e,Wn,d),f(e,ie,d),t(ie,os),t(ie,en),t(en,rs),t(ie,ls),f(e,Xn,d),f(e,W,d),t(W,pe),t(pe,tn),J(xe,tn,null),t(W,is),t(W,nn),t(nn,ps),f(e,Dn,d),f(e,N,d),J($e,N,null),t(N,ds),t(N,X),t(X,cs),t(X,mt),t(mt,hs),t(X,ms),t(X,Ee),t(Ee,us),t(X,fs),t(N,gs),t(N,D),t(D,_s),t(D,ut),t(ut,Ts),t(D,ys),t(D,ft),t(ft,bs),t(D,Js),f(e,Yn,d),f(e,Y,d),t(Y,de),t(de,an),J(Ce,an,null),t(Y,Ms),t(Y,sn),t(sn,Us),f(e,Ln,d),f(e,S,d),J(Pe,S,null),t(S,ks),t(S,on),t(on,js),t(S,vs),t(S,L),t(L,rn),t(rn,ws),t(L,Ss),t(L,ln),t(ln,Gs),t(L,Vs),t(L,Ne),t(Ne,xs),t(Ne,Fe),t(Fe,$s),t(Ne,Es),t(S,Cs),J(ce,S,null),t(S,Ps),J(he,S,null),t(S,Ns),J(me,S,null),t(S,Fs),t(S,ue),J(Re,ue,null),t(ue,Rs),t(ue,pn),t(pn,Is),t(S,zs),t(S,F),J(Ie,F,null),t(F,Qs),t(F,dn),t(dn,As),t(F,qs),J(fe,F,null),f(e,Hn,d),f(e,H,d),t(H,ge),t(ge,cn),J(ze,cn,null),t(H,Zs),t(H,hn),t(hn,Bs),f(e,Kn,d),f(e,E,d),J(Qe,E,null),t(E,Os),t(E,mn),t(mn,Ws),t(E,Xs),t(E,Ae),t(Ae,Ds),t(Ae,qe),t(qe,Ys),t(Ae,Ls),t(E,Hs),t(E,Ze),t(Ze,Ks),t(Ze,Be),t(Be,eo),t(Ze,to),t(E,no),t(E,R),J(Oe,R,null),t(R,ao),t(R,K),t(K,so),t(K,gt),t(gt,oo),t(K,ro),t(K,un),t(un,lo),t(K,io),t(R,po),J(_e,R,null),f(e,ea,d),f(e,ee,d),t(ee,Te),t(Te,fn),J(We,fn,null),t(ee,co),t(ee,gn),t(gn,ho),f(e,ta,d),f(e,C,d),J(Xe,C,null),t(C,mo),t(C,_n),t(_n,uo),t(C,fo),t(C,De),t(De,go),t(De,Ye),t(Ye,_o),t(De,To),t(C,yo),t(C,Le),t(Le,bo),t(Le,He),t(He,Jo),t(Le,Mo),t(C,Uo),t(C,v),J(Ke,v,null),t(v,ko),t(v,te),t(te,jo),t(te,_t),t(_t,vo),t(te,wo),t(te,Tn),t(Tn,So),t(te,Go),t(v,Vo),J(ye,v,null),t(v,xo),t(v,yn),t(yn,$o),t(v,Eo),t(v,bn),t(bn,Co),t(v,Po),J(be,v,null),t(v,No),t(v,Jn),t(Jn,Fo),t(v,Ro),J(Je,v,null),t(v,Io),t(v,Mn),t(Mn,zo),t(v,Qo),J(Me,v,null),na=!0},p(e,[d]){const et={};d&2&&(et.$$scope={dirty:d,ctx:e}),ce.$set(et);const Un={};d&2&&(Un.$$scope={dirty:d,ctx:e}),he.$set(Un);const kn={};d&2&&(kn.$$scope={dirty:d,ctx:e}),me.$set(kn);const jn={};d&2&&(jn.$$scope={dirty:d,ctx:e}),fe.$set(jn);const tt={};d&2&&(tt.$$scope={dirty:d,ctx:e}),_e.$set(tt);const vn={};d&2&&(vn.$$scope={dirty:d,ctx:e}),ye.$set(vn);const wn={};d&2&&(wn.$$scope={dirty:d,ctx:e}),be.$set(wn);const Sn={};d&2&&(Sn.$$scope={dirty:d,ctx:e}),Je.$set(Sn);const Gn={};d&2&&(Gn.$$scope={dirty:d,ctx:e}),Me.$set(Gn)},i(e){na||(M(i.$$.fragment,e),M(je.$$.fragment,e),M(ve.$$.fragment,e),M(we.$$.fragment,e),M(Se.$$.fragment,e),M(Ge.$$.fragment,e),M(Ve.$$.fragment,e),M(xe.$$.fragment,e),M($e.$$.fragment,e),M(Ce.$$.fragment,e),M(Pe.$$.fragment,e),M(ce.$$.fragment,e),M(he.$$.fragment,e),M(me.$$.fragment,e),M(Re.$$.fragment,e),M(Ie.$$.fragment,e),M(fe.$$.fragment,e),M(ze.$$.fragment,e),M(Qe.$$.fragment,e),M(Oe.$$.fragment,e),M(_e.$$.fragment,e),M(We.$$.fragment,e),M(Xe.$$.fragment,e),M(Ke.$$.fragment,e),M(ye.$$.fragment,e),M(be.$$.fragment,e),M(Je.$$.fragment,e),M(Me.$$.fragment,e),na=!0)},o(e){U(i.$$.fragment,e),U(je.$$.fragment,e),U(ve.$$.fragment,e),U(we.$$.fragment,e),U(Se.$$.fragment,e),U(Ge.$$.fragment,e),U(Ve.$$.fragment,e),U(xe.$$.fragment,e),U($e.$$.fragment,e),U(Ce.$$.fragment,e),U(Pe.$$.fragment,e),U(ce.$$.fragment,e),U(he.$$.fragment,e),U(me.$$.fragment,e),U(Re.$$.fragment,e),U(Ie.$$.fragment,e),U(fe.$$.fragment,e),U(ze.$$.fragment,e),U(Qe.$$.fragment,e),U(Oe.$$.fragment,e),U(_e.$$.fragment,e),U(We.$$.fragment,e),U(Xe.$$.fragment,e),U(Ke.$$.fragment,e),U(ye.$$.fragment,e),U(be.$$.fragment,e),U(Je.$$.fragment,e),U(Me.$$.fragment,e),na=!1},d(e){n(p),e&&n(T),e&&n(m),k(i),e&&n(Vn),e&&n(A),k(je),e&&n(xn),e&&n(st),e&&n($n),e&&n(ot),e&&n(En),e&&n(q),k(ve),e&&n(Cn),e&&n(se),e&&n(Pn),k(we,e),e&&n(Nn),e&&n(Z),k(Se),e&&n(Fn),e&&n(rt),e&&n(Rn),e&&n(B),k(Ge),e&&n(In),e&&n(w),e&&n(zn),e&&n(lt),e&&n(Qn),e&&n(it),e&&n(An),e&&n(pt),e&&n(qn),e&&n(dt),e&&n(Zn),e&&n(ct),e&&n(Bn),e&&n(ht),e&&n(On),e&&n(O),k(Ve),e&&n(Wn),e&&n(ie),e&&n(Xn),e&&n(W),k(xe),e&&n(Dn),e&&n(N),k($e),e&&n(Yn),e&&n(Y),k(Ce),e&&n(Ln),e&&n(S),k(Pe),k(ce),k(he),k(me),k(Re),k(Ie),k(fe),e&&n(Hn),e&&n(H),k(ze),e&&n(Kn),e&&n(E),k(Qe),k(Oe),k(_e),e&&n(ea),e&&n(ee),k(We),e&&n(ta),e&&n(C),k(Xe),k(Ke),k(ye),k(be),k(Je),k(Me)}}}const jl={local:"gptsanjapanese",sections:[{local:"overview",sections:[{local:"generation",title:"Generation"}],title:"Overview"},{local:"gptsan-features",sections:[{local:"prefixlm-model",title:"Prefix-LM Model"},{local:"spout-vector",title:"Spout Vector"}],title:"GPTSAN Features"},{local:"transformers.GPTSanJapaneseConfig",title:"GPTSanJapaneseConfig"},{local:"transformers.GPTSanJapaneseTokenizer",title:"GPTSanJapaneseTokenizer"},{local:"transformers.GPTSanJapaneseModel",title:"GPTSanJapaneseModel"},{local:"transformers.GPTSanJapaneseForConditionalGeneration",title:"GPTSanJapaneseForConditionalGeneration"}],title:"GPTSAN-japanese"};function vl(G){return ul(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class El extends dl{constructor(p){super();cl(this,p,vl,kl,hl,{})}}export{El as default,jl as metadata};
