import{S as mo,i as ho,s as fo,e as o,k as p,w as v,t as r,M as uo,c as a,d as s,m as c,a as n,x as T,h as l,b as m,G as t,g as d,y as E,q as M,o as y,B as j,v as _o,L as ko}from"../../chunks/vendor-hf-doc-builder.js";import{D as Se}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Bs}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as $s}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as go}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function bo(Pe){let u,C,k,_,J;return _=new Bs({props:{code:"MCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMEElN0MlMjBmaXJzdCUyMHNlcXVlbmNlJTIwJTIwJTIwJTIwJTdDJTIwc2Vjb25kJTIwc2VxdWVuY2UlMjAlN0M=",highlighted:`0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1 1
| first sequence    | second sequence |`}}),{c(){u=o("p"),C=r("pair mask has the following format:"),k=p(),v(_.$$.fragment)},l(h){u=a(h,"P",{});var z=n(u);C=l(z,"pair mask has the following format:"),z.forEach(s),k=c(h),T(_.$$.fragment,h)},m(h,z){d(h,u,z),t(u,C),d(h,k,z),E(_,h,z),J=!0},p:ko,i(h){J||(M(_.$$.fragment,h),J=!0)},o(h){y(_.$$.fragment,h),J=!1},d(h){h&&s(u),h&&s(k),j(_,h)}}}function wo(Pe){let u,C,k,_,J,h,z,Te,ut,Ne,B,D,Ee,Q,_t,Me,kt,Oe,pe,gt,Ze,ce,bt,Qe,U,I,wt,V,vt,Tt,X,Et,Mt,yt,ye,jt,Ve,b,Jt,je,zt,$t,Je,Bt,It,ze,xt,qt,Xe,A,Ct,W,Dt,Ut,We,de,At,Ge,G,Fe,me,Lt,He,F,Ye,he,Rt,Ke,fe,H,St,ue,Pt,Nt,et,L,Ot,Y,Zt,Qt,tt,x,R,$e,K,Vt,Be,Xt,st,f,ee,Wt,Ie,Gt,Ft,te,Ht,_e,Yt,Kt,es,$,se,ts,xe,ss,os,oe,ke,as,qe,ns,rs,ge,ls,Ce,is,ps,S,ae,cs,De,ds,ms,w,ne,hs,Ue,fs,us,P,_s,q,ks,Ae,gs,bs,Le,ws,vs,Ts,N,re,Es,le,Ms,Re,ys,js,ot;return h=new $s({}),Q=new $s({}),G=new Bs({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQWJlcnRqYXBhbmVzZSUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIyY2wtdG9ob2t1JTJGYmVydC1iYXNlLWphcGFuZXNlJTIyKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmNsLXRvaG9rdSUyRmJlcnQtYmFzZS1qYXBhbmVzZSUyMiklMEElMEElMjMlMjMlMjBJbnB1dCUyMEphcGFuZXNlJTIwVGV4dCUwQWxpbmUlMjAlM0QlMjAlMjIlRTUlOTAlQkUlRTglQkMlQTklRTMlODElQUYlRTclOEMlQUIlRTMlODElQTclRTMlODElODIlRTMlODIlOEIlRTMlODAlODIlMjIlMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIobGluZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQlNUIwJTVEKSklMEElMEFvdXRwdXRzJTIwJTNEJTIwYmVydGphcGFuZXNlKCoqaW5wdXRzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>bertjapanese = AutoModel.from_pretrained(<span class="hljs-string">&quot;cl-tohoku/bert-base-japanese&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;cl-tohoku/bert-base-japanese&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment">## Input Japanese Text</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>line = <span class="hljs-string">&quot;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u3002&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(line, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>][<span class="hljs-number">0</span>]))
[CLS] \u543E\u8F29 \u306F \u732B \u3067 \u3042\u308B \u3002 [SEP]

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = bertjapanese(**inputs)`}}),F=new Bs({props:{code:"YmVydGphcGFuZXNlJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJjbC10b2hva3UlMkZiZXJ0LWJhc2UtamFwYW5lc2UtY2hhciUyMiklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJjbC10b2hva3UlMkZiZXJ0LWJhc2UtamFwYW5lc2UtY2hhciUyMiklMEElMEElMjMlMjMlMjBJbnB1dCUyMEphcGFuZXNlJTIwVGV4dCUwQWxpbmUlMjAlM0QlMjAlMjIlRTUlOTAlQkUlRTglQkMlQTklRTMlODElQUYlRTclOEMlQUIlRTMlODElQTclRTMlODElODIlRTMlODIlOEIlRTMlODAlODIlMjIlMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIobGluZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQlNUIwJTVEKSklMEElMEFvdXRwdXRzJTIwJTNEJTIwYmVydGphcGFuZXNlKCoqaW5wdXRzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>bertjapanese = AutoModel.from_pretrained(<span class="hljs-string">&quot;cl-tohoku/bert-base-japanese-char&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;cl-tohoku/bert-base-japanese-char&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment">## Input Japanese Text</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>line = <span class="hljs-string">&quot;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u3002&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(line, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>][<span class="hljs-number">0</span>]))
[CLS] \u543E \u8F29 \u306F \u732B \u3067 \u3042 \u308B \u3002 [SEP]

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = bertjapanese(**inputs)`}}),K=new $s({}),ee=new Se({props:{name:"class transformers.BertJapaneseTokenizer",anchor:"transformers.BertJapaneseTokenizer",parameters:[{name:"vocab_file",val:""},{name:"spm_file",val:" = None"},{name:"do_lower_case",val:" = False"},{name:"do_word_tokenize",val:" = True"},{name:"do_subword_tokenize",val:" = True"},{name:"word_tokenizer_type",val:" = 'basic'"},{name:"subword_tokenizer_type",val:" = 'wordpiece'"},{name:"never_split",val:" = None"},{name:"unk_token",val:" = '[UNK]'"},{name:"sep_token",val:" = '[SEP]'"},{name:"pad_token",val:" = '[PAD]'"},{name:"cls_token",val:" = '[CLS]'"},{name:"mask_token",val:" = '[MASK]'"},{name:"mecab_kwargs",val:" = None"},{name:"sudachi_kwargs",val:" = None"},{name:"jumanpp_kwargs",val:" = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BertJapaneseTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to a one-wordpiece-per-line vocabulary file.`,name:"vocab_file"},{anchor:"transformers.BertJapaneseTokenizer.spm_file",description:`<strong>spm_file</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Path to <a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a .spm or .model
extension) that contains the vocabulary.`,name:"spm_file"},{anchor:"transformers.BertJapaneseTokenizer.do_lower_case",description:`<strong>do_lower_case</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to lower case the input. Only has an effect when do_basic_tokenize=True.`,name:"do_lower_case"},{anchor:"transformers.BertJapaneseTokenizer.do_word_tokenize",description:`<strong>do_word_tokenize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to do word tokenization.`,name:"do_word_tokenize"},{anchor:"transformers.BertJapaneseTokenizer.do_subword_tokenize",description:`<strong>do_subword_tokenize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to do subword tokenization.`,name:"do_subword_tokenize"},{anchor:"transformers.BertJapaneseTokenizer.word_tokenizer_type",description:`<strong>word_tokenizer_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;basic&quot;</code>) &#x2014;
Type of word tokenizer. Choose from [&#x201C;basic&#x201D;, &#x201C;mecab&#x201D;, &#x201C;sudachi&#x201D;, &#x201C;jumanpp&#x201D;].`,name:"word_tokenizer_type"},{anchor:"transformers.BertJapaneseTokenizer.subword_tokenizer_type",description:`<strong>subword_tokenizer_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;wordpiece&quot;</code>) &#x2014;
Type of subword tokenizer. Choose from [&#x201C;wordpiece&#x201D;, &#x201C;character&#x201D;, &#x201C;sentencepiece&#x201D;,].`,name:"subword_tokenizer_type"},{anchor:"transformers.BertJapaneseTokenizer.mecab_kwargs",description:`<strong>mecab_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary passed to the <code>MecabTokenizer</code> constructor.`,name:"mecab_kwargs"},{anchor:"transformers.BertJapaneseTokenizer.sudachi_kwargs",description:`<strong>sudachi_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary passed to the <code>SudachiTokenizer</code> constructor.`,name:"sudachi_kwargs"},{anchor:"transformers.BertJapaneseTokenizer.jumanpp_kwargs",description:`<strong>jumanpp_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary passed to the <code>JumanppTokenizer</code> constructor.`,name:"jumanpp_kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/bert_japanese/tokenization_bert_japanese.py#L107"}}),se=new Se({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.BertJapaneseTokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.BertJapaneseTokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.BertJapaneseTokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/bert_japanese/tokenization_bert_japanese.py#L308",returnDescription:`
<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),ae=new Se({props:{name:"convert_tokens_to_string",anchor:"transformers.BertJapaneseTokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/bert_japanese/tokenization_bert_japanese.py#L300"}}),ne=new Se({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.BertJapaneseTokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.BertJapaneseTokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.BertJapaneseTokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/bert_japanese/tokenization_bert_japanese.py#L363",returnDescription:`
<p>List of <a href="../glossary#token-type-ids">token type IDs</a> according to the given sequence(s).</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),P=new go({props:{anchor:"transformers.BertJapaneseTokenizer.create_token_type_ids_from_sequences.example",$$slots:{default:[bo]},$$scope:{ctx:Pe}}}),re=new Se({props:{name:"get_special_tokens_mask",anchor:"transformers.BertJapaneseTokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.BertJapaneseTokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.BertJapaneseTokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.BertJapaneseTokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.31.0/src/transformers/models/bert_japanese/tokenization_bert_japanese.py#L334",returnDescription:`
<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),{c(){u=o("meta"),C=p(),k=o("h1"),_=o("a"),J=o("span"),v(h.$$.fragment),z=p(),Te=o("span"),ut=r("BertJapanese"),Ne=p(),B=o("h2"),D=o("a"),Ee=o("span"),v(Q.$$.fragment),_t=p(),Me=o("span"),kt=r("Overview"),Oe=p(),pe=o("p"),gt=r("The BERT models trained on Japanese text."),Ze=p(),ce=o("p"),bt=r("There are models with two different tokenization methods:"),Qe=p(),U=o("ul"),I=o("li"),wt=r("Tokenize with MeCab and WordPiece. This requires some extra dependencies, "),V=o("a"),vt=r("fugashi"),Tt=r(" which is a wrapper around "),X=o("a"),Et=r("MeCab"),Mt=r("."),yt=p(),ye=o("li"),jt=r("Tokenize into characters."),Ve=p(),b=o("p"),Jt=r("To use "),je=o("em"),zt=r("MecabTokenizer"),$t=r(", you should "),Je=o("code"),Bt=r('pip install transformers["ja"]'),It=r(" (or "),ze=o("code"),xt=r('pip install -e .["ja"]'),qt=r(` if you install
from source) to install dependencies.`),Xe=p(),A=o("p"),Ct=r("See "),W=o("a"),Dt=r("details on cl-tohoku repository"),Ut=r("."),We=p(),de=o("p"),At=r("Example of using a model with MeCab and WordPiece tokenization:"),Ge=p(),v(G.$$.fragment),Fe=p(),me=o("p"),Lt=r("Example of using a model with Character tokenization:"),He=p(),v(F.$$.fragment),Ye=p(),he=o("p"),Rt=r("Tips:"),Ke=p(),fe=o("ul"),H=o("li"),St=r("This implementation is the same as BERT, except for tokenization method. Refer to the "),ue=o("a"),Pt=r("documentation of BERT"),Nt=r(" for more usage examples."),et=p(),L=o("p"),Ot=r("This model was contributed by "),Y=o("a"),Zt=r("cl-tohoku"),Qt=r("."),tt=p(),x=o("h2"),R=o("a"),$e=o("span"),v(K.$$.fragment),Vt=p(),Be=o("span"),Xt=r("BertJapaneseTokenizer"),st=p(),f=o("div"),v(ee.$$.fragment),Wt=p(),Ie=o("p"),Gt=r("Construct a BERT tokenizer for Japanese text."),Ft=p(),te=o("p"),Ht=r("This tokenizer inherits from "),_e=o("a"),Yt=r("PreTrainedTokenizer"),Kt=r(` which contains most of the main methods. Users should refer
to: this superclass for more information regarding those methods.`),es=p(),$=o("div"),v(se.$$.fragment),ts=p(),xe=o("p"),ss=r(`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A BERT sequence has the following format:`),os=p(),oe=o("ul"),ke=o("li"),as=r("single sequence: "),qe=o("code"),ns=r("[CLS] X [SEP]"),rs=p(),ge=o("li"),ls=r("pair of sequences: "),Ce=o("code"),is=r("[CLS] A [SEP] B [SEP]"),ps=p(),S=o("div"),v(ae.$$.fragment),cs=p(),De=o("p"),ds=r("Converts a sequence of tokens (string) in a single string."),ms=p(),w=o("div"),v(ne.$$.fragment),hs=p(),Ue=o("p"),fs=r("Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence"),us=p(),v(P.$$.fragment),_s=p(),q=o("p"),ks=r("If "),Ae=o("code"),gs=r("token_ids_1"),bs=r(" is "),Le=o("code"),ws=r("None"),vs=r(", this method only returns the first portion of the mask (0s)."),Ts=p(),N=o("div"),v(re.$$.fragment),Es=p(),le=o("p"),Ms=r(`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `),Re=o("code"),ys=r("prepare_for_model"),js=r(" method."),this.h()},l(e){const i=uo('[data-svelte="svelte-1phssyn"]',document.head);u=a(i,"META",{name:!0,content:!0}),i.forEach(s),C=c(e),k=a(e,"H1",{class:!0});var ie=n(k);_=a(ie,"A",{id:!0,class:!0,href:!0});var Is=n(_);J=a(Is,"SPAN",{});var xs=n(J);T(h.$$.fragment,xs),xs.forEach(s),Is.forEach(s),z=c(ie),Te=a(ie,"SPAN",{});var qs=n(Te);ut=l(qs,"BertJapanese"),qs.forEach(s),ie.forEach(s),Ne=c(e),B=a(e,"H2",{class:!0});var at=n(B);D=a(at,"A",{id:!0,class:!0,href:!0});var Cs=n(D);Ee=a(Cs,"SPAN",{});var Ds=n(Ee);T(Q.$$.fragment,Ds),Ds.forEach(s),Cs.forEach(s),_t=c(at),Me=a(at,"SPAN",{});var Us=n(Me);kt=l(Us,"Overview"),Us.forEach(s),at.forEach(s),Oe=c(e),pe=a(e,"P",{});var As=n(pe);gt=l(As,"The BERT models trained on Japanese text."),As.forEach(s),Ze=c(e),ce=a(e,"P",{});var Ls=n(ce);bt=l(Ls,"There are models with two different tokenization methods:"),Ls.forEach(s),Qe=c(e),U=a(e,"UL",{});var nt=n(U);I=a(nt,"LI",{});var be=n(I);wt=l(be,"Tokenize with MeCab and WordPiece. This requires some extra dependencies, "),V=a(be,"A",{href:!0,rel:!0});var Rs=n(V);vt=l(Rs,"fugashi"),Rs.forEach(s),Tt=l(be," which is a wrapper around "),X=a(be,"A",{href:!0,rel:!0});var Ss=n(X);Et=l(Ss,"MeCab"),Ss.forEach(s),Mt=l(be,"."),be.forEach(s),yt=c(nt),ye=a(nt,"LI",{});var Ps=n(ye);jt=l(Ps,"Tokenize into characters."),Ps.forEach(s),nt.forEach(s),Ve=c(e),b=a(e,"P",{});var O=n(b);Jt=l(O,"To use "),je=a(O,"EM",{});var Ns=n(je);zt=l(Ns,"MecabTokenizer"),Ns.forEach(s),$t=l(O,", you should "),Je=a(O,"CODE",{});var Os=n(Je);Bt=l(Os,'pip install transformers["ja"]'),Os.forEach(s),It=l(O," (or "),ze=a(O,"CODE",{});var Zs=n(ze);xt=l(Zs,'pip install -e .["ja"]'),Zs.forEach(s),qt=l(O,` if you install
from source) to install dependencies.`),O.forEach(s),Xe=c(e),A=a(e,"P",{});var rt=n(A);Ct=l(rt,"See "),W=a(rt,"A",{href:!0,rel:!0});var Qs=n(W);Dt=l(Qs,"details on cl-tohoku repository"),Qs.forEach(s),Ut=l(rt,"."),rt.forEach(s),We=c(e),de=a(e,"P",{});var Vs=n(de);At=l(Vs,"Example of using a model with MeCab and WordPiece tokenization:"),Vs.forEach(s),Ge=c(e),T(G.$$.fragment,e),Fe=c(e),me=a(e,"P",{});var Xs=n(me);Lt=l(Xs,"Example of using a model with Character tokenization:"),Xs.forEach(s),He=c(e),T(F.$$.fragment,e),Ye=c(e),he=a(e,"P",{});var Ws=n(he);Rt=l(Ws,"Tips:"),Ws.forEach(s),Ke=c(e),fe=a(e,"UL",{});var Gs=n(fe);H=a(Gs,"LI",{});var lt=n(H);St=l(lt,"This implementation is the same as BERT, except for tokenization method. Refer to the "),ue=a(lt,"A",{href:!0});var Fs=n(ue);Pt=l(Fs,"documentation of BERT"),Fs.forEach(s),Nt=l(lt," for more usage examples."),lt.forEach(s),Gs.forEach(s),et=c(e),L=a(e,"P",{});var it=n(L);Ot=l(it,"This model was contributed by "),Y=a(it,"A",{href:!0,rel:!0});var Hs=n(Y);Zt=l(Hs,"cl-tohoku"),Hs.forEach(s),Qt=l(it,"."),it.forEach(s),tt=c(e),x=a(e,"H2",{class:!0});var pt=n(x);R=a(pt,"A",{id:!0,class:!0,href:!0});var Ys=n(R);$e=a(Ys,"SPAN",{});var Ks=n($e);T(K.$$.fragment,Ks),Ks.forEach(s),Ys.forEach(s),Vt=c(pt),Be=a(pt,"SPAN",{});var eo=n(Be);Xt=l(eo,"BertJapaneseTokenizer"),eo.forEach(s),pt.forEach(s),st=c(e),f=a(e,"DIV",{class:!0});var g=n(f);T(ee.$$.fragment,g),Wt=c(g),Ie=a(g,"P",{});var to=n(Ie);Gt=l(to,"Construct a BERT tokenizer for Japanese text."),to.forEach(s),Ft=c(g),te=a(g,"P",{});var ct=n(te);Ht=l(ct,"This tokenizer inherits from "),_e=a(ct,"A",{href:!0});var so=n(_e);Yt=l(so,"PreTrainedTokenizer"),so.forEach(s),Kt=l(ct,` which contains most of the main methods. Users should refer
to: this superclass for more information regarding those methods.`),ct.forEach(s),es=c(g),$=a(g,"DIV",{class:!0});var we=n($);T(se.$$.fragment,we),ts=c(we),xe=a(we,"P",{});var oo=n(xe);ss=l(oo,`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A BERT sequence has the following format:`),oo.forEach(s),os=c(we),oe=a(we,"UL",{});var dt=n(oe);ke=a(dt,"LI",{});var Js=n(ke);as=l(Js,"single sequence: "),qe=a(Js,"CODE",{});var ao=n(qe);ns=l(ao,"[CLS] X [SEP]"),ao.forEach(s),Js.forEach(s),rs=c(dt),ge=a(dt,"LI",{});var zs=n(ge);ls=l(zs,"pair of sequences: "),Ce=a(zs,"CODE",{});var no=n(Ce);is=l(no,"[CLS] A [SEP] B [SEP]"),no.forEach(s),zs.forEach(s),dt.forEach(s),we.forEach(s),ps=c(g),S=a(g,"DIV",{class:!0});var mt=n(S);T(ae.$$.fragment,mt),cs=c(mt),De=a(mt,"P",{});var ro=n(De);ds=l(ro,"Converts a sequence of tokens (string) in a single string."),ro.forEach(s),mt.forEach(s),ms=c(g),w=a(g,"DIV",{class:!0});var Z=n(w);T(ne.$$.fragment,Z),hs=c(Z),Ue=a(Z,"P",{});var lo=n(Ue);fs=l(lo,"Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence"),lo.forEach(s),us=c(Z),T(P.$$.fragment,Z),_s=c(Z),q=a(Z,"P",{});var ve=n(q);ks=l(ve,"If "),Ae=a(ve,"CODE",{});var io=n(Ae);gs=l(io,"token_ids_1"),io.forEach(s),bs=l(ve," is "),Le=a(ve,"CODE",{});var po=n(Le);ws=l(po,"None"),po.forEach(s),vs=l(ve,", this method only returns the first portion of the mask (0s)."),ve.forEach(s),Z.forEach(s),Ts=c(g),N=a(g,"DIV",{class:!0});var ht=n(N);T(re.$$.fragment,ht),Es=c(ht),le=a(ht,"P",{});var ft=n(le);Ms=l(ft,`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `),Re=a(ft,"CODE",{});var co=n(Re);ys=l(co,"prepare_for_model"),co.forEach(s),js=l(ft," method."),ft.forEach(s),ht.forEach(s),g.forEach(s),this.h()},h(){m(u,"name","hf:doc:metadata"),m(u,"content",JSON.stringify(vo)),m(_,"id","bertjapanese"),m(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(_,"href","#bertjapanese"),m(k,"class","relative group"),m(D,"id","overview"),m(D,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(D,"href","#overview"),m(B,"class","relative group"),m(V,"href","https://github.com/polm/fugashi"),m(V,"rel","nofollow"),m(X,"href","https://taku910.github.io/mecab/"),m(X,"rel","nofollow"),m(W,"href","https://github.com/cl-tohoku/bert-japanese"),m(W,"rel","nofollow"),m(ue,"href","bert"),m(Y,"href","https://huggingface.co/cl-tohoku"),m(Y,"rel","nofollow"),m(R,"id","transformers.BertJapaneseTokenizer"),m(R,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(R,"href","#transformers.BertJapaneseTokenizer"),m(x,"class","relative group"),m(_e,"href","/docs/transformers/v4.31.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizer"),m($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,i){t(document.head,u),d(e,C,i),d(e,k,i),t(k,_),t(_,J),E(h,J,null),t(k,z),t(k,Te),t(Te,ut),d(e,Ne,i),d(e,B,i),t(B,D),t(D,Ee),E(Q,Ee,null),t(B,_t),t(B,Me),t(Me,kt),d(e,Oe,i),d(e,pe,i),t(pe,gt),d(e,Ze,i),d(e,ce,i),t(ce,bt),d(e,Qe,i),d(e,U,i),t(U,I),t(I,wt),t(I,V),t(V,vt),t(I,Tt),t(I,X),t(X,Et),t(I,Mt),t(U,yt),t(U,ye),t(ye,jt),d(e,Ve,i),d(e,b,i),t(b,Jt),t(b,je),t(je,zt),t(b,$t),t(b,Je),t(Je,Bt),t(b,It),t(b,ze),t(ze,xt),t(b,qt),d(e,Xe,i),d(e,A,i),t(A,Ct),t(A,W),t(W,Dt),t(A,Ut),d(e,We,i),d(e,de,i),t(de,At),d(e,Ge,i),E(G,e,i),d(e,Fe,i),d(e,me,i),t(me,Lt),d(e,He,i),E(F,e,i),d(e,Ye,i),d(e,he,i),t(he,Rt),d(e,Ke,i),d(e,fe,i),t(fe,H),t(H,St),t(H,ue),t(ue,Pt),t(H,Nt),d(e,et,i),d(e,L,i),t(L,Ot),t(L,Y),t(Y,Zt),t(L,Qt),d(e,tt,i),d(e,x,i),t(x,R),t(R,$e),E(K,$e,null),t(x,Vt),t(x,Be),t(Be,Xt),d(e,st,i),d(e,f,i),E(ee,f,null),t(f,Wt),t(f,Ie),t(Ie,Gt),t(f,Ft),t(f,te),t(te,Ht),t(te,_e),t(_e,Yt),t(te,Kt),t(f,es),t(f,$),E(se,$,null),t($,ts),t($,xe),t(xe,ss),t($,os),t($,oe),t(oe,ke),t(ke,as),t(ke,qe),t(qe,ns),t(oe,rs),t(oe,ge),t(ge,ls),t(ge,Ce),t(Ce,is),t(f,ps),t(f,S),E(ae,S,null),t(S,cs),t(S,De),t(De,ds),t(f,ms),t(f,w),E(ne,w,null),t(w,hs),t(w,Ue),t(Ue,fs),t(w,us),E(P,w,null),t(w,_s),t(w,q),t(q,ks),t(q,Ae),t(Ae,gs),t(q,bs),t(q,Le),t(Le,ws),t(q,vs),t(f,Ts),t(f,N),E(re,N,null),t(N,Es),t(N,le),t(le,Ms),t(le,Re),t(Re,ys),t(le,js),ot=!0},p(e,[i]){const ie={};i&2&&(ie.$$scope={dirty:i,ctx:e}),P.$set(ie)},i(e){ot||(M(h.$$.fragment,e),M(Q.$$.fragment,e),M(G.$$.fragment,e),M(F.$$.fragment,e),M(K.$$.fragment,e),M(ee.$$.fragment,e),M(se.$$.fragment,e),M(ae.$$.fragment,e),M(ne.$$.fragment,e),M(P.$$.fragment,e),M(re.$$.fragment,e),ot=!0)},o(e){y(h.$$.fragment,e),y(Q.$$.fragment,e),y(G.$$.fragment,e),y(F.$$.fragment,e),y(K.$$.fragment,e),y(ee.$$.fragment,e),y(se.$$.fragment,e),y(ae.$$.fragment,e),y(ne.$$.fragment,e),y(P.$$.fragment,e),y(re.$$.fragment,e),ot=!1},d(e){s(u),e&&s(C),e&&s(k),j(h),e&&s(Ne),e&&s(B),j(Q),e&&s(Oe),e&&s(pe),e&&s(Ze),e&&s(ce),e&&s(Qe),e&&s(U),e&&s(Ve),e&&s(b),e&&s(Xe),e&&s(A),e&&s(We),e&&s(de),e&&s(Ge),j(G,e),e&&s(Fe),e&&s(me),e&&s(He),j(F,e),e&&s(Ye),e&&s(he),e&&s(Ke),e&&s(fe),e&&s(et),e&&s(L),e&&s(tt),e&&s(x),j(K),e&&s(st),e&&s(f),j(ee),j(se),j(ae),j(ne),j(P),j(re)}}}const vo={local:"bertjapanese",sections:[{local:"overview",title:"Overview"},{local:"transformers.BertJapaneseTokenizer",title:"BertJapaneseTokenizer"}],title:"BertJapanese"};function To(Pe){return _o(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class zo extends mo{constructor(u){super();ho(this,u,To,wo,fo,{})}}export{zo as default,vo as metadata};
