

import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


__UpperCAmelCase     :   str    				=     logging.get_logger(__name__)

__UpperCAmelCase     :   int    				=     {'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

__UpperCAmelCase     :   Any    				=     {
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

__UpperCAmelCase     :   List[str]    				=     {
    'abeja/gpt-neox-japanese-2.7b': 2_0_4_8,
}







def 	_snake_case		(    SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE				)       ->      str:

     """simple docstring"""







     with open(SCREAMING_SNAKE_CASE		,							"r"		,							encoding="utf-8"				) as f:
          _lowerCAmelCase   :					List[str]        =    json.loads(f.read()				)

     _lowerCAmelCase   :					str        =    collections.OrderedDict()
     _lowerCAmelCase   :					List[str]        =    collections.OrderedDict()
     _lowerCAmelCase   :					List[Any]        =    collections.OrderedDict()
     with open(SCREAMING_SNAKE_CASE		,							"r"		,							encoding="utf-8"				) as f:
          _lowerCAmelCase   :					List[Any]        =    f.readlines()
     _lowerCAmelCase   :					List[Any]        =    [[t.rstrip("\n"				)] if (t == "," or "," not in t) else t.rstrip("\n"				).split(","				) for t in token]
     for idx, b in enumerate(SCREAMING_SNAKE_CASE				):
          _lowerCAmelCase   :					Union[str, Any]        =    b
          _lowerCAmelCase   :					Optional[Any]        =    idx
          for wd in b:
               _lowerCAmelCase   :					int        =    idx

     return vocab, raw_vocab, ids_to_tokens, emoji





class A__    ( A ):



 """simple docstring"""







 _lowercase     :					str      =						VOCAB_FILES_NAMES
 _lowercase     :					str      =						PRETRAINED_VOCAB_FILES_MAP
 _lowercase     :					Dict      =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowercase     :					int      =						['''input_ids''', '''attention_mask''']

 def __init__(		self :	Optional[int]     ,						A_ :	Union[str, Any]     ,						A_ :	Any     ,						A_ :	Union[str, Any]="<|endoftext|>"     ,						A_ :	Optional[int]="<|endoftext|>"     ,						A_ :	List[str]="<|startoftext|>"     ,						A_ :	Optional[Any]="<|endoftext|>"     ,						A_ :	int=False     ,						**A_ :	List[str]     ,						):






      '''simple docstring'''




      super().__init__(
          unk_token=A_     ,						pad_token=A_     ,						bos_token=A_     ,						eos_token=A_     ,						do_clean_text=A_     ,						**A_     ,						)
      if not os.path.isfile(A_       ):
           raise ValueError(
               F'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
               " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"       )
      if not os.path.isfile(A_       ):
           raise ValueError(
               F'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
               " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"       )
      _lowerCAmelCase   :					Optional[Any]        =    do_clean_text
      _lowerCAmelCase   :					int        =    load_vocab_and_emoji(A_     ,						A_       )
      _lowerCAmelCase   :					str        =    SubWordJapaneseTokenizer(
          vocab=self.vocab     ,						ids_to_tokens=self.ids_to_tokens     ,						emoji=self.emoji       )

 @property
 def    __magic_name__       (		self :	Dict       ):






      '''simple docstring'''




      return len(self.raw_vocab       )

 def    __magic_name__       (		self :	List[Any]       ):






      '''simple docstring'''




      return dict(self.raw_vocab     ,						**self.added_tokens_encoder       )

 def    __magic_name__       (		self :	Union[str, Any]     ,						A_ :	str       ):






      '''simple docstring'''




      return self.subword_tokenizer.tokenize(A_     ,						clean=self.do_clean_text       )

 def    __magic_name__       (		self :	List[str]     ,						A_ :	Any       ):






      '''simple docstring'''




      return self.vocab.get(A_     ,						self.vocab.get(self.unk_token       )       )

 def    __magic_name__       (		self :	Optional[int]     ,						A_ :	int       ):






      '''simple docstring'''




      return self.subword_tokenizer.convert_id_to_token(A_       )

 def    __magic_name__       (		self :	Optional[int]     ,						A_ :	List[str]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					List[str]        =    "".join(A_       ).strip()
      return out_string

 def    __magic_name__       (		self :	List[Any]     ,						A_ :	"Conversation"       ):






      '''simple docstring'''




      _lowerCAmelCase   :					List[Any]        =    []
      for is_user, text in conversation.iter_texts():
           input_ids.extend(self.encode(A_     ,						add_special_tokens=A_       ) + [self.eos_token_id]       )

      if len(A_       ) > self.model_max_length:
           _lowerCAmelCase   :					Any        =    input_ids[-self.model_max_length :]
      return input_ids


 def    __magic_name__       (		self :	Tuple     ,						A_ :	str     ,						A_ :	Optional[str] = None       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Any        =    0
      if os.path.isdir(A_       ):
           _lowerCAmelCase   :					int        =    os.path.join(
               A_     ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )
           _lowerCAmelCase   :					str        =    os.path.join(
               A_     ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"]       )
      else:
           _lowerCAmelCase   :					int        =    (
               (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
           )
           _lowerCAmelCase   :					Union[str, Any]        =    (
               (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
           )
      with open(A_     ,						"w"     ,						encoding="utf-8"       ) as writer:
           for token_index, token in self.ids_to_tokens.items():
                if index != token_index:
                     logger.warning(
                         F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                         " Please check that the vocabulary is not corrupted!"       )
                     _lowerCAmelCase   :					Optional[int]        =    token_index
                writer.write(",".join(A_       ) + "\n"       )
                index += 1
      with open(A_     ,						"w"     ,						encoding="utf-8"       ) as writer:
           json.dump(self.emoji     ,						A_       )
      return vocab_file, emoji_file





class A__    ( A ):



 """simple docstring"""







 def __init__(		self :	int     ,						A_ :	Dict     ,						A_ :	Union[str, Any]     ,						A_ :	Any       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Tuple        =    vocab  # same as swe
      _lowerCAmelCase   :					Union[str, Any]        =    ids_to_tokens  # same as bpe
      _lowerCAmelCase   :					Optional[int]        =    emoji
      _lowerCAmelCase   :					Tuple        =    np.max([len(A_       ) for w in self.vocab.keys()]       )
      _lowerCAmelCase   :					Optional[Any]        =    re.compile(r"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"       )
      _lowerCAmelCase   :					Optional[Any]        =    re.compile(r"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"       )
      _lowerCAmelCase   :					Union[str, Any]        =    re.compile(r"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"       )
      _lowerCAmelCase   :					str        =    re.compile(
          r"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"       )
      _lowerCAmelCase   :					str        =    re.compile(
          r"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"       )
      _lowerCAmelCase   :					Optional[Any]        =    re.compile(
          r"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"       )
      _lowerCAmelCase   :					Optional[Any]        =    "─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
      _lowerCAmelCase   :					Union[str, Any]        =    "▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
      _lowerCAmelCase   :					List[Any]        =    str.maketrans({k: "<BLOCK>" for k in keisen + blocks}       )

 def __len__(		self :	Union[str, Any]       ):






      '''simple docstring'''




      return len(self.ids_to_tokens       )

 def    __magic_name__       (		self :	str     ,						A_ :	Tuple       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Any        =    self.content_repattera.sub("<URL>"     ,						A_       )
      _lowerCAmelCase   :					List[str]        =    self.content_repattera.sub("<EMAIL>"     ,						A_       )
      _lowerCAmelCase   :					Optional[int]        =    self.content_repattera.sub("<TEL>"     ,						A_       )
      _lowerCAmelCase   :					str        =    self.content_repattera.sub("<DATE>"     ,						A_       )
      _lowerCAmelCase   :					List[str]        =    self.content_repattera.sub("<DATE>"     ,						A_       )
      _lowerCAmelCase   :					Optional[Any]        =    self.content_repattera.sub("<PRICE>"     ,						A_       )
      _lowerCAmelCase   :					Union[str, Any]        =    content.translate(self.content_transa       )
      while "<BLOCK><BLOCK>" in content:
           _lowerCAmelCase   :					Union[str, Any]        =    content.replace("<BLOCK><BLOCK>"     ,						"<BLOCK>"       )
      return content

 def    __magic_name__       (		self :	List[str]     ,						A_ :	Union[str, Any]     ,						A_ :	List[str]=False       ):






      '''simple docstring'''




      _lowerCAmelCase   :					int        =    text.replace(" "     ,						"<SP>"       )
      _lowerCAmelCase   :					List[str]        =    text.replace("　"     ,						"<SP>"       )
      _lowerCAmelCase   :					Union[str, Any]        =    text.replace("\r\n"     ,						"<BR>"       )
      _lowerCAmelCase   :					Optional[Any]        =    text.replace("\n"     ,						"<BR>"       )
      _lowerCAmelCase   :					Optional[int]        =    text.replace("\r"     ,						"<BR>"       )
      _lowerCAmelCase   :					int        =    text.replace("\t"     ,						"<TAB>"       )
      _lowerCAmelCase   :					Dict        =    text.replace("—"     ,						"ー"       )
      _lowerCAmelCase   :					int        =    text.replace("−"     ,						"ー"       )
      for k, v in self.emoji["emoji"].items():
           if k in text:
                _lowerCAmelCase   :					Any        =    text.replace(A_     ,						A_       )
      if clean:
           _lowerCAmelCase   :					Tuple        =    self.clean_text(A_       )

      def check_simbol(A_ :	str       ):
           _lowerCAmelCase   :					str        =    x.encode()
           if len(A_       ) == 1 and len(A_       ) == 2:
                _lowerCAmelCase   :					Optional[Any]        =    (int(e[0]       ) << 8) + int(e[1]       )
                if (
                    (c >= 0Xc_2_a_1 and c <= 0Xc_2_b_f)
                    or (c >= 0Xc_7_8_0 and c <= 0Xc_7_8_3)
                    or (c >= 0Xc_a_b_9 and c <= 0Xc_b_b_f)
                    or (c >= 0Xc_c_8_0 and c <= 0Xc_d_a_2)
                ):
                     return True
           return False

      def checkuae(A_ :	Union[str, Any]       ):
           _lowerCAmelCase   :					Optional[int]        =    x.encode()
           if len(A_       ) == 1 and len(A_       ) == 3:
                _lowerCAmelCase   :					Optional[int]        =    (int(e[0]       ) << 1_6) + (int(e[1]       ) << 8) + int(e[2]       )
                if c >= 0Xe_2_8_0_8_0 and c <= 0Xe_2_b_0_7_f:
                     return True
           return False

      _lowerCAmelCase   :					int        =    0
      _lowerCAmelCase   :					Union[str, Any]        =    []
      while pos < len(A_       ):
           _lowerCAmelCase   :					int        =    min(len(A_       )     ,						pos + self.maxlen + 1       ) if text[pos] == "<" else pos + 3
           _lowerCAmelCase   :					Tuple        =    []  # (token_id, token, pos)
           for e in range(A_     ,						A_     ,						-1       ):
                _lowerCAmelCase   :					List[Any]        =    text[pos:e]
                if wd in self.vocab:
                     if wd[0] == "<" and len(A_       ) > 2:
                          _lowerCAmelCase   :					int        =    [(self.vocab[wd], wd, e)]
                          break
                     else:
                          candidates.append((self.vocab[wd], wd, e)       )
           if len(A_       ) > 0:
                # the smallest token_id is adopted
                _lowerCAmelCase   :					Tuple        =    sorted(A_     ,						key=lambda A_       : x[0]       )[0]
                result.append(A_       )
                _lowerCAmelCase   :					int        =    e
           else:
                _lowerCAmelCase   :					str        =    pos + 1
                _lowerCAmelCase   :					Any        =    text[pos:end]
                if check_simbol(A_       ):
                     result.append("<KIGOU>"       )
                elif checkuae(A_       ):
                     result.append("<U2000U2BFF>"       )
                else:
                     for i in wd.encode("utf-8"       ):
                          result.append("<|byte%d|>" % i       )
                _lowerCAmelCase   :					Union[str, Any]        =    end
      return result


 def    __magic_name__       (		self :	str     ,						A_ :	List[str]     ,						A_ :	List[str]="\n"       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Optional[Any]        =    []
      _lowerCAmelCase   :					Tuple        =    []
      _lowerCAmelCase   :					Union[str, Any]        =    self.ids_to_tokens[index][0]
      if word[:6] == "<|byte" and word[-2:] == "|>":
           byte_tokens.append(int(word[6:-2]       )       )
      else:
           if len(A_       ) > 0:
                words.append(bytearray(A_       ).decode("utf-8"     ,						errors="replace"       )       )
                _lowerCAmelCase   :					Optional[int]        =    []
           if word[:7] == "<|emoji" and word[-2:] == "|>":
                words.append(self.emoji["emoji_inv"][word]       )
           elif word == "<SP>":
                words.append(" "       )
           elif word == "<BR>":
                words.append(A_       )
           elif word == "<TAB>":
                words.append("\t"       )
           elif word == "<BLOCK>":
                words.append("▀"       )
           elif word == "<KIGOU>":
                words.append("ǀ"       )
           elif word == "<U2000U2BFF>":
                words.append("‖"       )
           else:
                words.append(A_       )
      if len(A_       ) > 0:
           words.append(bytearray(A_       ).decode("utf-8"     ,						errors="replace"       )       )
      _lowerCAmelCase   :					List[Any]        =    "".join(A_       )
      return text









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tensorflow_text_available, is_torch_available


__UpperCAmelCase    				=     {
    'configuration_ernie': ['ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ErnieConfig', 'ErnieOnnxConfig'],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __UpperCAmelCase    				=     [
      'ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST',
      'ErnieForCausalLM',
      'ErnieForMaskedLM',
      'ErnieForMultipleChoice',
      'ErnieForNextSentencePrediction',
      'ErnieForPreTraining',
      'ErnieForQuestionAnswering',
      'ErnieForSequenceClassification',
      'ErnieForTokenClassification',
      'ErnieModel',
      'ErniePreTrainedModel',
  ]

if TYPE_CHECKING:
  from .configuration_ernie import ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieConfig, ErnieOnnxConfig

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_ernie import (
        ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST,
        ErnieForCausalLM,
        ErnieForMaskedLM,
        ErnieForMultipleChoice,
        ErnieForNextSentencePrediction,
        ErnieForPreTraining,
        ErnieForQuestionAnswering,
        ErnieForSequenceClassification,
        ErnieForTokenClassification,
        ErnieModel,
        ErniePreTrainedModel,
    )

else:
  import sys

  __UpperCAmelCase    				=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






