





'''simple docstring'''

import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


a        =						logging.get_logger(__name__)

a        =						{"vocab_file": "vocab.txt", "emoji_file": "emoji.json"}

a        =						{
    "vocab_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt",
    },
    "emoji_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json",
    },
}

a        =						{
    "abeja/gpt-neox-japanese-2.7b": 2048,
}

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase      )						->					Tuple:

   '''simple docstring'''







   with open(__UpperCAmelCase							,	"""r"""							,	encoding="""utf-8"""      ) as f:
      __SCREAMING_SNAKE_CASE      							=       json.loads(f.read()      )

   __SCREAMING_SNAKE_CASE      							=       collections.OrderedDict()
   __SCREAMING_SNAKE_CASE      							=       collections.OrderedDict()
   __SCREAMING_SNAKE_CASE      							=       collections.OrderedDict()
   with open(__UpperCAmelCase							,	"""r"""							,	encoding="""utf-8"""      ) as f:
      __SCREAMING_SNAKE_CASE      							=       f.readlines()
   __SCREAMING_SNAKE_CASE      							=       [[t.rstrip("""\n"""      )] if (t == """,""" or """,""" not in t) else t.rstrip("""\n"""      ).split(""","""      ) for t in token]
   for idx, b in enumerate(__UpperCAmelCase      ):
      __SCREAMING_SNAKE_CASE      							=       b
      __SCREAMING_SNAKE_CASE      							=       idx
      for wd in b:
         __SCREAMING_SNAKE_CASE      							=       idx

   return vocab, raw_vocab, ids_to_tokens, emoji







class __a (   _snake_case			):
   __UpperCamelCase   : Tuple	      =	VOCAB_FILES_NAMES
   __UpperCamelCase   : Optional[int]	      =	PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase   : List[Any]	      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase   : Optional[Any]	      =	['input_ids', 'attention_mask']





   def __init__(					self				:			Optional[int]				,lowerCamelCase				:			Any				,lowerCamelCase				:			int				,lowerCamelCase				:			Union[str, Any]="<|endoftext|>"				,lowerCamelCase				:			Optional[Any]="<|endoftext|>"				,lowerCamelCase				:			Any="<|startoftext|>"				,lowerCamelCase				:			int="<|endoftext|>"				,lowerCamelCase				:			Dict=False				,**lowerCamelCase				:			Optional[Any]				,):

      '''simple docstring'''

      super().__init__(
          unk_token=lowerCamelCase				,pad_token=lowerCamelCase				,bos_token=lowerCamelCase				,eos_token=lowerCamelCase				,do_clean_text=lowerCamelCase				,**lowerCamelCase				,)
      if not os.path.isfile(lowerCamelCase    ):
         raise ValueError(
             f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
             """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""    )
      if not os.path.isfile(lowerCamelCase    ):
         raise ValueError(
             f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
             """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""    )
      __SCREAMING_SNAKE_CASE      							=       do_clean_text
      __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       load_vocab_and_emoji(lowerCamelCase				,lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       SubWordJapaneseTokenizer(
          vocab=self.vocab				,ids_to_tokens=self.ids_to_tokens				,emoji=self.emoji    )





   @property
   def      UpperCAmelCase__     (					self				:			Union[str, Any]    ):

      '''simple docstring'''

      return len(self.raw_vocab    )





   def      UpperCAmelCase__     (					self				:			Tuple    ):

      '''simple docstring'''

      return dict(self.raw_vocab				,**self.added_tokens_encoder    )





   def      UpperCAmelCase__     (					self				:			List[Any]				,lowerCamelCase				:			Optional[Any]    ):

      '''simple docstring'''

      return self.subword_tokenizer.tokenize(lowerCamelCase				,clean=self.do_clean_text    )





   def      UpperCAmelCase__     (					self				:			Any				,lowerCamelCase				:			Tuple    ):

      '''simple docstring'''

      return self.vocab.get(lowerCamelCase				,self.vocab.get(self.unk_token    )    )





   def      UpperCAmelCase__     (					self				:			Optional[int]				,lowerCamelCase				:			Union[str, Any]    ):

      '''simple docstring'''

      return self.subword_tokenizer.convert_id_to_token(lowerCamelCase    )





   def      UpperCAmelCase__     (					self				:			int				,lowerCamelCase				:			Optional[Any]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       """""".join(lowerCamelCase    ).strip()
      return out_string





   def      UpperCAmelCase__     (					self				:			Union[str, Any]				,lowerCamelCase				:			"Conversation"    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       []
      for is_user, text in conversation.iter_texts():
         input_ids.extend(self.encode(lowerCamelCase				,add_special_tokens=lowerCamelCase    ) + [self.eos_token_id]    )

      if len(lowerCamelCase    ) > self.model_max_length:
         __SCREAMING_SNAKE_CASE      							=       input_ids[-self.model_max_length :]
      return input_ids





   def      UpperCAmelCase__     (					self				:			List[Any]				,lowerCamelCase				:			str				,lowerCamelCase				:			Optional[str] = None    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       0
      if os.path.isdir(lowerCamelCase    ):
         __SCREAMING_SNAKE_CASE      							=       os.path.join(
             lowerCamelCase				,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
         __SCREAMING_SNAKE_CASE      							=       os.path.join(
             lowerCamelCase				,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""]    )
      else:
         __SCREAMING_SNAKE_CASE      							=       (
             (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""vocab_file"""]
         )
         __SCREAMING_SNAKE_CASE      							=       (
             (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""emoji_file"""]
         )
      with open(lowerCamelCase				,"""w"""				,encoding="""utf-8"""    ) as writer:
         for token_index, token in self.ids_to_tokens.items():
            if index != token_index:
               logger.warning(
                   f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                   """ Please check that the vocabulary is not corrupted!"""    )
               __SCREAMING_SNAKE_CASE      							=       token_index
            writer.write(""",""".join(lowerCamelCase    ) + """\n"""    )
            index += 1
      with open(lowerCamelCase				,"""w"""				,encoding="""utf-8"""    ) as writer:
         json.dump(self.emoji				,lowerCamelCase    )
      return vocab_file, emoji_file







class __a (   _snake_case			):





   def __init__(					self				:			Dict				,lowerCamelCase				:			List[Any]				,lowerCamelCase				:			Union[str, Any]				,lowerCamelCase				:			Optional[int]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       vocab  # same as swe
      __SCREAMING_SNAKE_CASE      							=       ids_to_tokens  # same as bpe
      __SCREAMING_SNAKE_CASE      							=       emoji
      __SCREAMING_SNAKE_CASE      							=       np.max([len(lowerCamelCase    ) for w in self.vocab.keys()]    )
      __SCREAMING_SNAKE_CASE      							=       re.compile(r"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"""    )
      __SCREAMING_SNAKE_CASE      							=       re.compile(r"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"""    )
      __SCREAMING_SNAKE_CASE      							=       re.compile(r"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"""    )
      __SCREAMING_SNAKE_CASE      							=       re.compile(
          r"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""    )
      __SCREAMING_SNAKE_CASE      							=       re.compile(
          r"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""    )
      __SCREAMING_SNAKE_CASE      							=       re.compile(
          r"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"""    )
      __SCREAMING_SNAKE_CASE      							=       """─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"""
      __SCREAMING_SNAKE_CASE      							=       """▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"""
      __SCREAMING_SNAKE_CASE      							=       str.maketrans({k: """<BLOCK>""" for k in keisen + blocks}    )





   def __len__(					self				:			Tuple    ):

      '''simple docstring'''

      return len(self.ids_to_tokens    )





   def      UpperCAmelCase__     (					self				:			Any				,lowerCamelCase				:			Union[str, Any]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       self.content_repattera.sub("""<URL>"""				,lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       self.content_repattera.sub("""<EMAIL>"""				,lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       self.content_repattera.sub("""<TEL>"""				,lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       self.content_repattera.sub("""<DATE>"""				,lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       self.content_repattera.sub("""<DATE>"""				,lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       self.content_repattera.sub("""<PRICE>"""				,lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       content.translate(self.content_transa    )
      while "<BLOCK><BLOCK>" in content:
         __SCREAMING_SNAKE_CASE      							=       content.replace("""<BLOCK><BLOCK>"""				,"""<BLOCK>"""    )
      return content





   def      UpperCAmelCase__     (					self				:			List[Any]				,lowerCamelCase				:			Optional[Any]				,lowerCamelCase				:			Optional[Any]=False    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       text.replace(""" """				,"""<SP>"""    )
      __SCREAMING_SNAKE_CASE      							=       text.replace("""　"""				,"""<SP>"""    )
      __SCREAMING_SNAKE_CASE      							=       text.replace("""\r\n"""				,"""<BR>"""    )
      __SCREAMING_SNAKE_CASE      							=       text.replace("""\n"""				,"""<BR>"""    )
      __SCREAMING_SNAKE_CASE      							=       text.replace("""\r"""				,"""<BR>"""    )
      __SCREAMING_SNAKE_CASE      							=       text.replace("""\t"""				,"""<TAB>"""    )
      __SCREAMING_SNAKE_CASE      							=       text.replace("""—"""				,"""ー"""    )
      __SCREAMING_SNAKE_CASE      							=       text.replace("""−"""				,"""ー"""    )
      for k, v in self.emoji["emoji"].items():
         if k in text:
            __SCREAMING_SNAKE_CASE      							=       text.replace(lowerCamelCase				,lowerCamelCase    )
      if clean:
         __SCREAMING_SNAKE_CASE      							=       self.clean_text(lowerCamelCase    )

      def check_simbol(lowerCamelCase				:			List[str]    ):
         __SCREAMING_SNAKE_CASE      							=       x.encode()
         if len(lowerCamelCase    ) == 1 and len(lowerCamelCase    ) == 2:
            __SCREAMING_SNAKE_CASE      							=       (int(e[0]    ) << 8) + int(e[1]    )
            if (
                (c >= 0xC_2A1 and c <= 0xC_2BF)
                or (c >= 0xC_780 and c <= 0xC_783)
                or (c >= 0xC_AB9 and c <= 0xC_BBF)
                or (c >= 0xC_C80 and c <= 0xC_DA2)
            ):
               return True
         return False

      def checkuae(lowerCamelCase				:			Optional[Any]    ):
         __SCREAMING_SNAKE_CASE      							=       x.encode()
         if len(lowerCamelCase    ) == 1 and len(lowerCamelCase    ) == 3:
            __SCREAMING_SNAKE_CASE      							=       (int(e[0]    ) << 16) + (int(e[1]    ) << 8) + int(e[2]    )
            if c >= 0xE28_080 and c <= 0xE2B_07F:
               return True
         return False

      __SCREAMING_SNAKE_CASE      							=       0
      __SCREAMING_SNAKE_CASE      							=       []
      while pos < len(lowerCamelCase    ):
         __SCREAMING_SNAKE_CASE      							=       min(len(lowerCamelCase    )				,pos + self.maxlen + 1    ) if text[pos] == """<""" else pos + 3
         __SCREAMING_SNAKE_CASE      							=       []  # (token_id, token, pos)
         for e in range(lowerCamelCase				,lowerCamelCase				,-1    ):
            __SCREAMING_SNAKE_CASE      							=       text[pos:e]
            if wd in self.vocab:
               if wd[0] == "<" and len(lowerCamelCase    ) > 2:
                  __SCREAMING_SNAKE_CASE      							=       [(self.vocab[wd], wd, e)]
                  break
               else:
                  candidates.append((self.vocab[wd], wd, e)    )
         if len(lowerCamelCase    ) > 0:
            # the smallest token_id is adopted
            __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       sorted(lowerCamelCase				,key=lambda lowerCamelCase    : x[0]    )[0]
            result.append(lowerCamelCase    )
            __SCREAMING_SNAKE_CASE      							=       e
         else:
            __SCREAMING_SNAKE_CASE      							=       pos + 1
            __SCREAMING_SNAKE_CASE      							=       text[pos:end]
            if check_simbol(lowerCamelCase    ):
               result.append("""<KIGOU>"""    )
            elif checkuae(lowerCamelCase    ):
               result.append("""<U2000U2BFF>"""    )
            else:
               for i in wd.encode("""utf-8"""    ):
                  result.append("""<|byte%d|>""" % i    )
            __SCREAMING_SNAKE_CASE      							=       end
      return result





   def      UpperCAmelCase__     (					self				:			Union[str, Any]				,lowerCamelCase				:			Dict				,lowerCamelCase				:			str="\n"    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       []
      __SCREAMING_SNAKE_CASE      							=       []
      __SCREAMING_SNAKE_CASE      							=       self.ids_to_tokens[index][0]
      if word[:6] == "<|byte" and word[-2:] == "|>":
         byte_tokens.append(int(word[6:-2]    )    )
      else:
         if len(lowerCamelCase    ) > 0:
            words.append(bytearray(lowerCamelCase    ).decode("""utf-8"""				,errors="""replace"""    )    )
            __SCREAMING_SNAKE_CASE      							=       []
         if word[:7] == "<|emoji" and word[-2:] == "|>":
            words.append(self.emoji["""emoji_inv"""][word]    )
         elif word == "<SP>":
            words.append(""" """    )
         elif word == "<BR>":
            words.append(lowerCamelCase    )
         elif word == "<TAB>":
            words.append("""\t"""    )
         elif word == "<BLOCK>":
            words.append("""▀"""    )
         elif word == "<KIGOU>":
            words.append("""ǀ"""    )
         elif word == "<U2000U2BFF>":
            words.append("""‖"""    )
         else:
            words.append(lowerCamelCase    )
      if len(lowerCamelCase    ) > 0:
         words.append(bytearray(lowerCamelCase    ).decode("""utf-8"""				,errors="""replace"""    )    )
      __SCREAMING_SNAKE_CASE      							=       """""".join(lowerCamelCase    )
      return text












'''simple docstring'''

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

a        =						[
    "python",
    "tqdm",
    "regex",
    "requests",
    "packaging",
    "filelock",
    "numpy",
    "tokenizers",
    "huggingface-hub",
    "safetensors",
    "accelerate",
    "pyyaml",
]

for pkg in pkgs_to_check_at_runtime:
    if pkg in deps:
        if pkg == "tokenizers":
            # must be loaded here, or else tqdm check may fail
            from .utils import is_tokenizers_available

            if not is_tokenizers_available():
                continue  # not required, check version only if installed
        elif pkg == "accelerate":
            # must be loaded here, or else tqdm check may fail
            from .utils import is_accelerate_available

            # Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of
            # Transformers with PyTorch
            if not is_accelerate_available():
                continue  # not required, check version only if installed

        require_version_core(deps[pkg])
    else:
        raise ValueError(F'''can\'t find {pkg} in {deps.keys()}, check dependency_versions_table.py''')

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase=None      )						->					Optional[Any]:

   '''simple docstring'''







   require_version(deps[pkg]							,	__UpperCAmelCase      )





