






"""simple docstring"""





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


A__					: int=	logging.get_logger(__name__)

A__					: Optional[int]=	{"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

A__					: Optional[int]=	{
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

A__					: Optional[Any]=	{
    """abeja/gpt-neox-japanese-2.7b""": 20_48,
}







def   lowerCAmelCase_( SCREAMING_SNAKE_CASE ,	SCREAMING_SNAKE_CASE					)					->	str:
				"""simple docstring"""

				with open(_lowerCamelCase ,	'r' ,	encoding='utf-8'					) as f:
								UpperCamelCase__         =    json.loads(f.read()					)

				UpperCamelCase__         =    collections.OrderedDict()
				UpperCamelCase__         =    collections.OrderedDict()
				UpperCamelCase__         =    collections.OrderedDict()
				with open(_lowerCamelCase ,	'r' ,	encoding='utf-8'					) as f:
								UpperCamelCase__         =    f.readlines()
				UpperCamelCase__         =    [[t.rstrip('\n'					)] if (t == ',' or ',' not in t) else t.rstrip('\n'					).split(','					) for t in token]
				for idx, b in enumerate(_lowerCamelCase					):
								UpperCamelCase__         =    b
								UpperCamelCase__         =    idx
								for wd in b:
												UpperCamelCase__         =    idx

				return vocab, raw_vocab, ids_to_tokens, emoji
class    __lowerCamelCase    (  a__  ):
			a      :				Union[str, Any] 						=VOCAB_FILES_NAMES
			a      :				Dict 						=PRETRAINED_VOCAB_FILES_MAP
			a      :				str 						=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			a      :				str 						=["""input_ids""", """attention_mask"""]

			def __init__(					self ,				snake_case_ ,				snake_case_ ,				snake_case_="<|endoftext|>" ,				snake_case_="<|endoftext|>" ,				snake_case_="<|startoftext|>" ,				snake_case_="<|endoftext|>" ,				snake_case_=False ,				**snake_case_ ,				)		->       Dict:
							super().__init__(
							    unk_token=_A ,				pad_token=_A ,				bos_token=_A ,				eos_token=_A ,				do_clean_text=_A ,				**_A ,				)
							if not os.path.isfile(_A      ):
											raise ValueError(
											    F'Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'
											    ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'      )
							if not os.path.isfile(_A      ):
											raise ValueError(
											    F'Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'
											    ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'      )
							UpperCamelCase__         =    do_clean_text
							UpperCamelCase__         =    load_vocab_and_emoji(_A ,				_A      )
							UpperCamelCase__         =    SubWordJapaneseTokenizer(
							    vocab=self.vocab ,				ids_to_tokens=self.ids_to_tokens ,				emoji=self.emoji      )

			@property
			def 						SCREAMING_SNAKE_CASE__ (					self      )		->       Union[str, Any]:
							return len(self.raw_vocab      )

			def 						SCREAMING_SNAKE_CASE__ (					self      )		->       str:
							return dict(self.raw_vocab ,				**self.added_tokens_encoder      )

			def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       Dict:
							return self.subword_tokenizer.tokenize(_A ,				clean=self.do_clean_text      )

			def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       List[str]:
							return self.vocab.get(_A ,				self.vocab.get(self.unk_token      )      )

			def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       int:
							return self.subword_tokenizer.convert_id_to_token(_A      )

			def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       int:
							UpperCamelCase__         =    ''.join(_A      ).strip()
							return out_string

			def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       Optional[int]:
							UpperCamelCase__         =    []
							for is_user, text in conversation.iter_texts():
											input_ids.extend(self.encode(_A ,				add_special_tokens=_A      ) + [self.eos_token_id]      )

							if len(_A      ) > self.model_max_length:
											UpperCamelCase__         =    input_ids[-self.model_max_length :]
							return input_ids







			def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_ ,				snake_case_ = None      )		->       str:
							UpperCamelCase__         =    0
							if os.path.isdir(_A      ):
											UpperCamelCase__         =    os.path.join(
											    _A ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
											UpperCamelCase__         =    os.path.join(
											    _A ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']      )
							else:
											UpperCamelCase__         =    (
											    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
											)
											UpperCamelCase__         =    (
											    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
											)
							with open(_A ,				'w' ,				encoding='utf-8'      ) as writer:
											for token_index, token in self.ids_to_tokens.items():
															if index != token_index:
																			logger.warning(
																			    F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																			    ' Please check that the vocabulary is not corrupted!'      )
																			UpperCamelCase__         =    token_index
															writer.write(','.join(_A      ) + '\n'      )
															index += 1
							with open(_A ,				'w' ,				encoding='utf-8'      ) as writer:
											json.dump(self.emoji ,				_A      )
							return vocab_file, emoji_file







class    __lowerCamelCase    (  a__  ):

			def __init__(					self ,				snake_case_ ,				snake_case_ ,				snake_case_      )		->       Tuple:
							UpperCamelCase__         =    vocab  # same as swe
							UpperCamelCase__         =    ids_to_tokens  # same as bpe
							UpperCamelCase__         =    emoji
							UpperCamelCase__         =    np.max([len(_A      ) for w in self.vocab.keys()]      )
							UpperCamelCase__         =    re.compile(r'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'      )
							UpperCamelCase__         =    re.compile(r'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'      )
							UpperCamelCase__         =    re.compile(r'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'      )
							UpperCamelCase__         =    re.compile(
							    r'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'      )
							UpperCamelCase__         =    re.compile(
							    r'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'      )
							UpperCamelCase__         =    re.compile(
							    r'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'      )
							UpperCamelCase__         =    '─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
							UpperCamelCase__         =    '▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
							UpperCamelCase__         =    str.maketrans({k: '<BLOCK>' for k in keisen + blocks}      )

			def __len__(					self      )		->       Optional[Any]:
							return len(self.ids_to_tokens      )

			def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       Tuple:
							UpperCamelCase__         =    self.content_repattera.sub('<URL>' ,				_A      )
							UpperCamelCase__         =    self.content_repattera.sub('<EMAIL>' ,				_A      )
							UpperCamelCase__         =    self.content_repattera.sub('<TEL>' ,				_A      )
							UpperCamelCase__         =    self.content_repattera.sub('<DATE>' ,				_A      )
							UpperCamelCase__         =    self.content_repattera.sub('<DATE>' ,				_A      )
							UpperCamelCase__         =    self.content_repattera.sub('<PRICE>' ,				_A      )
							UpperCamelCase__         =    content.translate(self.content_transa      )
							while "<BLOCK><BLOCK>" in content:
											UpperCamelCase__         =    content.replace('<BLOCK><BLOCK>' ,				'<BLOCK>'      )
							return content

			def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_ ,				snake_case_=False      )		->       Union[str, Any]:
							UpperCamelCase__         =    text.replace(' ' ,				'<SP>'      )
							UpperCamelCase__         =    text.replace('　' ,				'<SP>'      )
							UpperCamelCase__         =    text.replace('\r\n' ,				'<BR>'      )
							UpperCamelCase__         =    text.replace('\n' ,				'<BR>'      )
							UpperCamelCase__         =    text.replace('\r' ,				'<BR>'      )
							UpperCamelCase__         =    text.replace('\t' ,				'<TAB>'      )
							UpperCamelCase__         =    text.replace('—' ,				'ー'      )
							UpperCamelCase__         =    text.replace('−' ,				'ー'      )
							for k, v in self.emoji["emoji"].items():
											if k in text:
															UpperCamelCase__         =    text.replace(_A ,				_A      )
							if clean:
											UpperCamelCase__         =    self.clean_text(_A      )

							def check_simbol(snake_case_      ):
											UpperCamelCase__         =    x.encode()
											if len(_A      ) == 1 and len(_A      ) == 2:
															UpperCamelCase__         =    (int(e[0]      ) << 8) + int(e[1]      )
															if (
															    (c >= 0Xc2a1 and c <= 0Xc2bf)
															    or (c >= 0Xc780 and c <= 0Xc783)
															    or (c >= 0Xcab9 and c <= 0Xcbbf)
															    or (c >= 0Xcc80 and c <= 0Xcda2)
															):
																			return True
											return False

							def checkuae(snake_case_      ):
											UpperCamelCase__         =    x.encode()
											if len(_A      ) == 1 and len(_A      ) == 3:
															UpperCamelCase__         =    (int(e[0]      ) << 16) + (int(e[1]      ) << 8) + int(e[2]      )
															if c >= 0Xe2_8080 and c <= 0Xe2_b07f:
																			return True
											return False

							UpperCamelCase__         =    0
							UpperCamelCase__         =    []
							while pos < len(_A      ):
											UpperCamelCase__         =    min(len(_A      ) ,				pos + self.maxlen + 1      ) if text[pos] == '<' else pos + 3
											UpperCamelCase__         =    []  # (token_id, token, pos)
											for e in range(_A ,				_A ,				-1      ):
															UpperCamelCase__         =    text[pos:e]
															if wd in self.vocab:
																			if wd[0] == "<" and len(_A      ) > 2:
																							UpperCamelCase__         =    [(self.vocab[wd], wd, e)]
																							break
																			else:
																							candidates.append((self.vocab[wd], wd, e)      )
											if len(_A      ) > 0:
															# the smallest token_id is adopted
															UpperCamelCase__         =    sorted(_A ,				key=lambda snake_case_      : x[0]      )[0]
															result.append(_A      )
															UpperCamelCase__         =    e
											else:
															UpperCamelCase__         =    pos + 1
															UpperCamelCase__         =    text[pos:end]
															if check_simbol(_A      ):
																			result.append('<KIGOU>'      )
															elif checkuae(_A      ):
																			result.append('<U2000U2BFF>'      )
															else:
																			for i in wd.encode('utf-8'      ):
																							result.append('<|byte%d|>' % i      )
															UpperCamelCase__         =    end
							return result







			def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_ ,				snake_case_="\n"      )		->       str:
							UpperCamelCase__         =    []
							UpperCamelCase__         =    []
							UpperCamelCase__         =    self.ids_to_tokens[index][0]
							if word[:6] == "<|byte" and word[-2:] == "|>":
											byte_tokens.append(int(word[6:-2]      )      )
							else:
											if len(_A      ) > 0:
															words.append(bytearray(_A      ).decode('utf-8' ,				errors='replace'      )      )
															UpperCamelCase__         =    []
											if word[:7] == "<|emoji" and word[-2:] == "|>":
															words.append(self.emoji['emoji_inv'][word]      )
											elif word == "<SP>":
															words.append(' '      )
											elif word == "<BR>":
															words.append(_A      )
											elif word == "<TAB>":
															words.append('\t'      )
											elif word == "<BLOCK>":
															words.append('▀'      )
											elif word == "<KIGOU>":
															words.append('ǀ'      )
											elif word == "<U2000U2BFF>":
															words.append('‖'      )
											else:
															words.append(_A      )
							if len(_A      ) > 0:
											words.append(bytearray(_A      ).decode('utf-8' ,				errors='replace'      )      )
							UpperCamelCase__         =    ''.join(_A      )
							return text












"""simple docstring"""





import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


A__					: Union[str, Any]=	logging.getLogger()
@unittest.skip("""Temporarily disable the doc tests."""  )
@require_torch
@require_tf
@slow
class    __lowerCamelCase    (  unittest.TestCase  ):

   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_ ,				snake_case_ = None ,				snake_case_ = None ,				snake_case_ = None ,				snake_case_ = True ,				)		->       Tuple:
       UpperCamelCase__         =    [file for file in os.listdir(snake_case_      ) if os.path.isfile(os.path.join(snake_case_ ,				snake_case_      )      )]

       if identifier is not None:
           UpperCamelCase__         =    [file for file in files if identifier in file]

       if n_identifier is not None:
           if isinstance(snake_case_ ,				snake_case_      ):
               for n_ in n_identifier:
                   UpperCamelCase__         =    [file for file in files if n_ not in file]
           else:
               UpperCamelCase__         =    [file for file in files if n_identifier not in file]

       UpperCamelCase__         =    ignore_files or []
       ignore_files.append('__init__.py'      )
       UpperCamelCase__         =    [file for file in files if file not in ignore_files]

       for file in files:
           # Open all files
           print('Testing' ,				snake_case_      )

           if only_modules:
               UpperCamelCase__         =    file.split('.'      )[0]
               try:
                   UpperCamelCase__         =    getattr(snake_case_ ,				snake_case_      )
                   UpperCamelCase__         =    doctest.DocTestSuite(snake_case_      )
                   UpperCamelCase__         =    unittest.TextTestRunner().run(snake_case_      )
                   self.assertIs(len(result.failures      ) ,				0      )
               except AttributeError:
                   logger.info(F'{module_identifier} is not a module.'      )
           else:
               UpperCamelCase__         =    doctest.testfile(str('..' / directory / file      ) ,				optionflags=doctest.ELLIPSIS      )
               self.assertIs(result.failed ,				0      )

   def 						SCREAMING_SNAKE_CASE__ (					self      )		->       Union[str, Any]:
       UpperCamelCase__         =    Path('src/transformers'      )
       UpperCamelCase__         =    'modeling'
       UpperCamelCase__         =    [
           'modeling_ctrl.py',
           'modeling_tf_ctrl.py',
       ]
       self.analyze_directory(snake_case_ ,				identifier=snake_case_ ,				ignore_files=snake_case_      )

   def 						SCREAMING_SNAKE_CASE__ (					self      )		->       Optional[int]:
       UpperCamelCase__         =    Path('src/transformers'      )
       UpperCamelCase__         =    'tokenization'
       self.analyze_directory(snake_case_ ,				identifier=snake_case_      )

   def 						SCREAMING_SNAKE_CASE__ (					self      )		->       Optional[Any]:
       UpperCamelCase__         =    Path('src/transformers'      )
       UpperCamelCase__         =    'configuration'
       self.analyze_directory(snake_case_ ,				identifier=snake_case_      )

   def 						SCREAMING_SNAKE_CASE__ (					self      )		->       str:
       UpperCamelCase__         =    Path('src/transformers'      )
       UpperCamelCase__         =    ['configuration', 'modeling', 'tokenization']
       self.analyze_directory(snake_case_ ,				n_identifier=snake_case_      )







   def 						SCREAMING_SNAKE_CASE__ (					self      )		->       Union[str, Any]:
       UpperCamelCase__         =    Path('docs/source'      )
       UpperCamelCase__         =    ['favicon.ico']
       self.analyze_directory(snake_case_ ,				ignore_files=snake_case_ ,				only_modules=snake_case_      )




