





"""simple docstring"""






import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


lowerCAmelCase_			=							logging.get_logger(__name__)

lowerCAmelCase_			=							{'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

lowerCAmelCase_			=							{
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

lowerCAmelCase_			=							{
    '''abeja/gpt-neox-japanese-2.7b''': 2_048,
}

def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       Dict:






       """simple docstring"""




       with open(_lowercase,'r',encoding='utf-8'							) as f:
              _UpperCAmelCase     =     json.loads(f.read()							)

       _UpperCAmelCase     =     collections.OrderedDict()
       _UpperCAmelCase     =     collections.OrderedDict()
       _UpperCAmelCase     =     collections.OrderedDict()
       with open(_lowercase,'r',encoding='utf-8'							) as f:
              _UpperCAmelCase     =     f.readlines()
       _UpperCAmelCase     =     [[t.rstrip('\n'							)] if (t == ',' or ',' not in t) else t.rstrip('\n'							).split(','							) for t in token]
       for idx, b in enumerate(_lowercase							):
              _UpperCAmelCase     =     b
              _UpperCAmelCase     =     idx
              for wd in b:
                     _UpperCAmelCase     =     idx

       return vocab, raw_vocab, ids_to_tokens, emoji



class 	lowerCAmelCase					(      snake_case							):
       lowerCAmelCase__						 =  VOCAB_FILES_NAMES
       lowerCAmelCase__						 =  PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase__						 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCAmelCase__						 =  ["""input_ids""", """attention_mask"""]






       def __init__(   self    ,  a__    ,  a__    ,  a__="<|endoftext|>"    ,  a__="<|endoftext|>"    ,  a__="<|startoftext|>"    ,  a__="<|endoftext|>"    ,  a__=False    ,  **a__    ,  ):
              super().__init__(
                  unk_token=UpperCamelCase_    ,  pad_token=UpperCamelCase_    ,  bos_token=UpperCamelCase_    ,  eos_token=UpperCamelCase_    ,  do_clean_text=UpperCamelCase_    ,  **UpperCamelCase_    ,  )
              if not os.path.isfile(UpperCamelCase_     ):
                     raise ValueError(
                         f"""Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained"""
                         ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'     )
              if not os.path.isfile(UpperCamelCase_     ):
                     raise ValueError(
                         f"""Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google"""
                         ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'     )
              _UpperCAmelCase     =     do_clean_text
              _UpperCAmelCase     =     load_vocab_and_emoji(UpperCamelCase_    ,  UpperCamelCase_     )
              _UpperCAmelCase     =     SubWordJapaneseTokenizer(
                  vocab=self.vocab    ,  ids_to_tokens=self.ids_to_tokens    ,  emoji=self.emoji     )






       @property
       def 							__A (   self     ):
              return len(self.raw_vocab     )






       def 							__A (   self     ):
              return dict(self.raw_vocab    ,  **self.added_tokens_encoder     )






       def 							__A (   self    ,  a__     ):
              return self.subword_tokenizer.tokenize(UpperCamelCase_    ,  clean=self.do_clean_text     )






       def 							__A (   self    ,  a__     ):
              return self.vocab.get(UpperCamelCase_    ,  self.vocab.get(self.unk_token     )     )






       def 							__A (   self    ,  a__     ):
              return self.subword_tokenizer.convert_id_to_token(UpperCamelCase_     )






       def 							__A (   self    ,  a__     ):
              _UpperCAmelCase     =     ''.join(UpperCamelCase_     ).strip()
              return out_string






       def 							__A (   self    ,  a__     ):
              _UpperCAmelCase     =     []
              for is_user, text in conversation.iter_texts():
                     input_ids.extend(self.encode(UpperCamelCase_    ,  add_special_tokens=UpperCamelCase_     ) + [self.eos_token_id]     )

              if len(UpperCamelCase_     ) > self.model_max_length:
                     _UpperCAmelCase     =     input_ids[-self.model_max_length :]
              return input_ids






       def 							__A (   self    ,  a__    ,  a__ = None     ):
              _UpperCAmelCase     =     0
              if os.path.isdir(UpperCamelCase_     ):
                     _UpperCAmelCase     =     os.path.join(
                         UpperCamelCase_    ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
                     _UpperCAmelCase     =     os.path.join(
                         UpperCamelCase_    ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']     )
              else:
                     _UpperCAmelCase     =     (
                         (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
                     )
                     _UpperCAmelCase     =     (
                         (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
                     )
              with open(UpperCamelCase_    ,  'w'    ,  encoding='utf-8'     ) as writer:
                     for token_index, token in self.ids_to_tokens.items():
                            if index != token_index:
                                   logger.warning(
                                       f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                                       ' Please check that the vocabulary is not corrupted!'     )
                                   _UpperCAmelCase     =     token_index
                            writer.write(','.join(UpperCamelCase_     ) + '\n'     )
                            index += 1
              with open(UpperCamelCase_    ,  'w'    ,  encoding='utf-8'     ) as writer:
                     json.dump(self.emoji    ,  UpperCamelCase_     )
              return vocab_file, emoji_file



class 	lowerCAmelCase					(      snake_case							):






       def __init__(   self    ,  a__    ,  a__    ,  a__     ):
              _UpperCAmelCase     =     vocab  # same as swe
              _UpperCAmelCase     =     ids_to_tokens  # same as bpe
              _UpperCAmelCase     =     emoji
              _UpperCAmelCase     =     np.max([len(UpperCamelCase_     ) for w in self.vocab.keys()]     )
              _UpperCAmelCase     =     re.compile(R'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'     )
              _UpperCAmelCase     =     re.compile(R'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'     )
              _UpperCAmelCase     =     re.compile(R'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'     )
              _UpperCAmelCase     =     re.compile(
                  R'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'     )
              _UpperCAmelCase     =     re.compile(
                  R'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'     )
              _UpperCAmelCase     =     re.compile(
                  R'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'     )
              _UpperCAmelCase     =     '─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
              _UpperCAmelCase     =     '▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
              _UpperCAmelCase     =     str.maketrans({k: '<BLOCK>' for k in keisen + blocks}     )






       def __len__(   self     ):
              return len(self.ids_to_tokens     )






       def 							__A (   self    ,  a__     ):
              _UpperCAmelCase     =     self.content_repattera.sub('<URL>'    ,  UpperCamelCase_     )
              _UpperCAmelCase     =     self.content_repattera.sub('<EMAIL>'    ,  UpperCamelCase_     )
              _UpperCAmelCase     =     self.content_repattera.sub('<TEL>'    ,  UpperCamelCase_     )
              _UpperCAmelCase     =     self.content_repattera.sub('<DATE>'    ,  UpperCamelCase_     )
              _UpperCAmelCase     =     self.content_repattera.sub('<DATE>'    ,  UpperCamelCase_     )
              _UpperCAmelCase     =     self.content_repattera.sub('<PRICE>'    ,  UpperCamelCase_     )
              _UpperCAmelCase     =     content.translate(self.content_transa     )
              while "<BLOCK><BLOCK>" in content:
                     _UpperCAmelCase     =     content.replace('<BLOCK><BLOCK>'    ,  '<BLOCK>'     )
              return content






       def 							__A (   self    ,  a__    ,  a__=False     ):
              _UpperCAmelCase     =     text.replace(' '    ,  '<SP>'     )
              _UpperCAmelCase     =     text.replace('　'    ,  '<SP>'     )
              _UpperCAmelCase     =     text.replace('\r\n'    ,  '<BR>'     )
              _UpperCAmelCase     =     text.replace('\n'    ,  '<BR>'     )
              _UpperCAmelCase     =     text.replace('\r'    ,  '<BR>'     )
              _UpperCAmelCase     =     text.replace('\t'    ,  '<TAB>'     )
              _UpperCAmelCase     =     text.replace('—'    ,  'ー'     )
              _UpperCAmelCase     =     text.replace('−'    ,  'ー'     )
              for k, v in self.emoji["emoji"].items():
                     if k in text:
                            _UpperCAmelCase     =     text.replace(UpperCamelCase_    ,  UpperCamelCase_     )
              if clean:
                     _UpperCAmelCase     =     self.clean_text(UpperCamelCase_     )

              def check_simbol(a__     ):
                     _UpperCAmelCase     =     x.encode()
                     if len(UpperCamelCase_     ) == 1 and len(UpperCamelCase_     ) == 2:
                            _UpperCAmelCase     =     (int(e[0]     ) << 8) + int(e[1]     )
                            if (
                                (c >= 0xc_2_a_1 and c <= 0xc_2_b_f)
                                or (c >= 0xc_7_8_0 and c <= 0xc_7_8_3)
                                or (c >= 0xc_a_b_9 and c <= 0xc_b_b_f)
                                or (c >= 0xc_c_8_0 and c <= 0xc_d_a_2)
                            ):
                                   return True
                     return False

              def checkuae(a__     ):
                     _UpperCAmelCase     =     x.encode()
                     if len(UpperCamelCase_     ) == 1 and len(UpperCamelCase_     ) == 3:
                            _UpperCAmelCase     =     (int(e[0]     ) << 16) + (int(e[1]     ) << 8) + int(e[2]     )
                            if c >= 0xe_2_8_0_8_0 and c <= 0xe_2_b_0_7_f:
                                   return True
                     return False

              _UpperCAmelCase     =     0
              _UpperCAmelCase     =     []
              while pos < len(UpperCamelCase_     ):
                     _UpperCAmelCase     =     min(len(UpperCamelCase_     )    ,  pos + self.maxlen + 1     ) if text[pos] == '<' else pos + 3
                     _UpperCAmelCase     =     []  # (token_id, token, pos)
                     for e in range(UpperCamelCase_    ,  UpperCamelCase_    ,  -1     ):
                            _UpperCAmelCase     =     text[pos:e]
                            if wd in self.vocab:
                                   if wd[0] == "<" and len(UpperCamelCase_     ) > 2:
                                          _UpperCAmelCase     =     [(self.vocab[wd], wd, e)]
                                          break
                                   else:
                                          candidates.append((self.vocab[wd], wd, e)     )
                     if len(UpperCamelCase_     ) > 0:
                            # the smallest token_id is adopted
                            _UpperCAmelCase     =     sorted(UpperCamelCase_    ,  key=lambda a__     : x[0]     )[0]
                            result.append(UpperCamelCase_     )
                            _UpperCAmelCase     =     e
                     else:
                            _UpperCAmelCase     =     pos + 1
                            _UpperCAmelCase     =     text[pos:end]
                            if check_simbol(UpperCamelCase_     ):
                                   result.append('<KIGOU>'     )
                            elif checkuae(UpperCamelCase_     ):
                                   result.append('<U2000U2BFF>'     )
                            else:
                                   for i in wd.encode('utf-8'     ):
                                          result.append('<|byte%d|>' % i     )
                            _UpperCAmelCase     =     end
              return result






       def 							__A (   self    ,  a__    ,  a__="\n"     ):
              _UpperCAmelCase     =     []
              _UpperCAmelCase     =     []
              _UpperCAmelCase     =     self.ids_to_tokens[index][0]
              if word[:6] == "<|byte" and word[-2:] == "|>":
                     byte_tokens.append(int(word[6:-2]     )     )
              else:
                     if len(UpperCamelCase_     ) > 0:
                            words.append(bytearray(UpperCamelCase_     ).decode('utf-8'    ,  errors='replace'     )     )
                            _UpperCAmelCase     =     []
                     if word[:7] == "<|emoji" and word[-2:] == "|>":
                            words.append(self.emoji['emoji_inv'][word]     )
                     elif word == "<SP>":
                            words.append(' '     )
                     elif word == "<BR>":
                            words.append(UpperCamelCase_     )
                     elif word == "<TAB>":
                            words.append('\t'     )
                     elif word == "<BLOCK>":
                            words.append('▀'     )
                     elif word == "<KIGOU>":
                            words.append('ǀ'     )
                     elif word == "<U2000U2BFF>":
                            words.append('‖'     )
                     else:
                            words.append(UpperCamelCase_     )
              if len(UpperCamelCase_     ) > 0:
                     words.append(bytearray(UpperCamelCase_     ).decode('utf-8'    ,  errors='replace'     )     )
              _UpperCAmelCase     =     ''.join(UpperCamelCase_     )
              return text













"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_			=							logging.get_logger(__name__)

lowerCAmelCase_			=							'''▁'''

lowerCAmelCase_			=							{'''vocab_file''': '''sentencepiece.bpe.model''', '''monolingual_vocab_file''': '''dict.txt'''}

lowerCAmelCase_			=							{
    '''vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model''',
    },
    '''monolingual_vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt''',
    },
}

lowerCAmelCase_			=							{'''vinai/bartpho-syllable''': 1_024}



class 	lowerCAmelCase					(      snake_case							):
       lowerCAmelCase__						 =  VOCAB_FILES_NAMES
       lowerCAmelCase__						 =  PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase__						 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCAmelCase__						 =  ["""input_ids""", """attention_mask"""]






       def __init__(   self    ,  a__    ,  a__    ,  a__="<s>"    ,  a__="</s>"    ,  a__="</s>"    ,  a__="<s>"    ,  a__="<unk>"    ,  a__="<pad>"    ,  a__="<mask>"    ,  a__ = None    ,  **a__    ,  ):
              # Mask token behave like a normal word, i.e. include the space before it
              _UpperCAmelCase     =     AddedToken(a__    ,  lstrip=a__    ,  rstrip=a__     ) if isinstance(a__    ,  a__     ) else mask_token

              _UpperCAmelCase     =     {} if sp_model_kwargs is None else sp_model_kwargs

              super().__init__(
                  bos_token=a__    ,  eos_token=a__    ,  unk_token=a__    ,  sep_token=a__    ,  cls_token=a__    ,  pad_token=a__    ,  mask_token=a__    ,  sp_model_kwargs=self.sp_model_kwargs    ,  **a__    ,  )

              _UpperCAmelCase     =     vocab_file
              _UpperCAmelCase     =     monolingual_vocab_file
              _UpperCAmelCase     =     spm.SentencePieceProcessor(**self.sp_model_kwargs     )
              self.sp_model.Load(str(a__     )     )

              # Load the reduced vocab

              # Keep order of special tokens for backward compatibility
              _UpperCAmelCase     =     {}
              _UpperCAmelCase     =     0
              for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
                     if str(a__     ) not in self.fairseq_tokens_to_ids:
                            _UpperCAmelCase     =     cnt
                            cnt += 1
              with open(a__    ,  'r'    ,  encoding='utf-8'     ) as f:
                     for line in f.readlines():
                            _UpperCAmelCase     =     line.strip().split()[0]
                            _UpperCAmelCase     =     len(self.fairseq_tokens_to_ids     )
              if str(a__     ) not in self.fairseq_tokens_to_ids:
                     _UpperCAmelCase     =     len(self.fairseq_tokens_to_ids     )

              _UpperCAmelCase     =     {v: k for k, v in self.fairseq_tokens_to_ids.items()}






       def __getstate__(   self     ):
              _UpperCAmelCase     =     self.__dict__.copy()
              _UpperCAmelCase     =     None
              _UpperCAmelCase     =     self.sp_model.serialized_model_proto()
              return state






       def __setstate__(   self    ,  a__     ):
              _UpperCAmelCase     =     d

              # for backward compatibility
              if not hasattr(self    ,  'sp_model_kwargs'     ):
                     _UpperCAmelCase     =     {}

              _UpperCAmelCase     =     spm.SentencePieceProcessor(**self.sp_model_kwargs     )
              self.sp_model.LoadFromSerializedProto(self.sp_model_proto     )






       def 							__A (   self    ,  a__    ,  a__ = None     ):

              if token_ids_a is None:
                     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
              _UpperCAmelCase     =     [self.cls_token_id]
              _UpperCAmelCase     =     [self.sep_token_id]
              return cls + token_ids_a + sep + sep + token_ids_a + sep






       def 							__A (   self    ,  a__    ,  a__ = None    ,  a__ = False     ):

              if already_has_special_tokens:
                     return super().get_special_tokens_mask(
                         token_ids_a=a__    ,  token_ids_a=a__    ,  already_has_special_tokens=a__     )

              if token_ids_a is None:
                     return [1] + ([0] * len(a__     )) + [1]
              return [1] + ([0] * len(a__     )) + [1, 1] + ([0] * len(a__     )) + [1]






       def 							__A (   self    ,  a__    ,  a__ = None     ):
              _UpperCAmelCase     =     [self.sep_token_id]
              _UpperCAmelCase     =     [self.cls_token_id]

              if token_ids_a is None:
                     return len(cls + token_ids_a + sep     ) * [0]
              return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]






       @property
       def 							__A (   self     ):
              return len(self.fairseq_ids_to_tokens     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     {self.convert_ids_to_tokens(a__     ): i for i in range(self.vocab_size     )}
              vocab.update(self.added_tokens_encoder     )
              return vocab






       def 							__A (   self    ,  a__     ):
              return self.sp_model.encode(a__    ,  out_type=a__     )






       def 							__A (   self    ,  a__     ):
              if token in self.fairseq_tokens_to_ids:
                     return self.fairseq_tokens_to_ids[token]
              else:
                     return self.unk_token_id






       def 							__A (   self    ,  a__     ):
              return self.fairseq_ids_to_tokens[index]






       def 							__A (   self    ,  a__     ):
              _UpperCAmelCase     =     ''.join(a__     ).replace(a__    ,  ' '     ).strip()
              return out_string






       def 							__A (   self    ,  a__    ,  a__ = None     ):
              if not os.path.isdir(a__     ):
                     logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
                     return
              _UpperCAmelCase     =     os.path.join(
                  a__    ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
              _UpperCAmelCase     =     os.path.join(
                  a__    ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['monolingual_vocab_file']    ,  )

              if os.path.abspath(self.vocab_file     ) != os.path.abspath(a__     ) and os.path.isfile(self.vocab_file     ):
                     copyfile(self.vocab_file    ,  a__     )
              elif not os.path.isfile(self.vocab_file     ):
                     with open(a__    ,  'wb'     ) as fi:
                            _UpperCAmelCase     =     self.sp_model.serialized_model_proto()
                            fi.write(a__     )

              if os.path.abspath(self.monolingual_vocab_file     ) != os.path.abspath(
                  a__     ) and os.path.isfile(self.monolingual_vocab_file     ):
                     copyfile(self.monolingual_vocab_file    ,  a__     )
              elif not os.path.isfile(self.monolingual_vocab_file     ):
                     with open(a__    ,  'w'    ,  encoding='utf-8'     ) as fp:
                            for token in self.fairseq_tokens_to_ids:
                                   if token not in self.all_special_tokens:
                                          fp.write(f"""{str(a__     )} \n"""     )

              return out_vocab_file, out_monolingual_vocab_file






