



'''simple docstring'''

import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


_A				:    Optional[Any]     						=logging.get_logger(__name__)

_A				:    int     						={'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

_A				:    Tuple     						={
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

_A				:    Optional[Any]     						={
    '''abeja/gpt-neox-japanese-2.7b''': 2_0_4_8,
}







def 						__UpperCamelCase				(							_lowercase,				_lowercase					) ->			Dict:
	with open(_lowercase,				'r',				encoding='utf-8'					) as f:
		_lowercase					:       Optional[Any]       =		json.loads(f.read()					)

	_lowercase					:       Union[str, Any]       =		collections.OrderedDict()
	_lowercase					:       List[Any]       =		collections.OrderedDict()
	_lowercase					:       str       =		collections.OrderedDict()
	with open(_lowercase,				'r',				encoding='utf-8'					) as f:
		_lowercase					:       Dict       =		f.readlines()
	_lowercase					:       str       =		[[t.rstrip('\n'					)] if (t == ',' or ',' not in t) else t.rstrip('\n'					).split(','					) for t in token]
	for idx, b in enumerate(_lowercase					):
		_lowercase					:       Tuple       =		b
		_lowercase					:       Any       =		idx
		for wd in b:
			_lowercase					:       Union[str, Any]       =		idx

	return vocab, raw_vocab, ids_to_tokens, emoji



class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					A_						    =						VOCAB_FILES_NAMES
					A_						    =						PRETRAINED_VOCAB_FILES_MAP
					A_						    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A_						    =						["""input_ids""", """attention_mask"""]




					def __init__(  self				:	List[Any]     ,  UpperCamelCase_				:	Tuple     ,  UpperCamelCase_				:	Optional[int]     ,  UpperCamelCase_				:	List[Any]="<|endoftext|>"     ,  UpperCamelCase_				:	Union[str, Any]="<|endoftext|>"     ,  UpperCamelCase_				:	Dict="<|startoftext|>"     ,  UpperCamelCase_				:	str="<|endoftext|>"     ,  UpperCamelCase_				:	int=False     ,  **UpperCamelCase_				:	str     ,  )  ->       Tuple:



						'''simple docstring'''





						super().__init__(
						    unk_token=UpperCamelCase_     ,  pad_token=UpperCamelCase_     ,  bos_token=UpperCamelCase_     ,  eos_token=UpperCamelCase_     ,  do_clean_text=UpperCamelCase_     ,  **UpperCamelCase_     ,  )
						if not os.path.isfile(UpperCamelCase_					):
							raise ValueError(
							    F'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
							    ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'					)
						if not os.path.isfile(UpperCamelCase_					):
							raise ValueError(
							    F'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
							    ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'					)
						_lowercase					:       Dict       =		do_clean_text
						_lowercase  ,    _lowercase  ,    _lowercase  ,    _lowercase					:       Dict       =		load_vocab_and_emoji(UpperCamelCase_     ,  UpperCamelCase_					)
						_lowercase					:       List[str]       =		SubWordJapaneseTokenizer(
						    vocab=self.vocab     ,  ids_to_tokens=self.ids_to_tokens     ,  emoji=self.emoji					)




					@property
					def 						__UpperCAmelCase     (  self				:	Any					)  ->       int:



						'''simple docstring'''





						return len(self.raw_vocab					)




					def 						__UpperCAmelCase     (  self				:	int					)  ->       Union[str, Any]:



						'''simple docstring'''





						return dict(self.raw_vocab     ,  **self.added_tokens_encoder					)




					def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	Optional[Any]					)  ->       Any:



						'''simple docstring'''





						return self.subword_tokenizer.tokenize(UpperCamelCase_     ,  clean=self.do_clean_text					)




					def 						__UpperCAmelCase     (  self				:	List[str]     ,  UpperCamelCase_				:	Any					)  ->       str:



						'''simple docstring'''





						return self.vocab.get(UpperCamelCase_     ,  self.vocab.get(self.unk_token					)					)




					def 						__UpperCAmelCase     (  self				:	int     ,  UpperCamelCase_				:	Tuple					)  ->       Union[str, Any]:



						'''simple docstring'''





						return self.subword_tokenizer.convert_id_to_token(UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	Any     ,  UpperCamelCase_				:	List[Any]					)  ->       str:



						'''simple docstring'''





						_lowercase					:       Any       =		''.join(UpperCamelCase_					).strip()
						return out_string




					def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	"Conversation"					)  ->       List[int]:



						'''simple docstring'''





						_lowercase					:       Optional[Any]       =		[]
						for is_user, text in conversation.iter_texts():
							input_ids.extend(self.encode(UpperCamelCase_     ,  add_special_tokens=UpperCamelCase_					) + [self.eos_token_id]					)

						if len(UpperCamelCase_					) > self.model_max_length:
							_lowercase					:       List[Any]       =		input_ids[-self.model_max_length :]
						return input_ids




					def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Optional[str] = None					)  ->       Tuple[str]:



						'''simple docstring'''





						_lowercase					:       Any       =		0
						if os.path.isdir(UpperCamelCase_					):
							_lowercase					:       Optional[Any]       =		os.path.join(
							    UpperCamelCase_     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
							_lowercase					:       Tuple       =		os.path.join(
							    UpperCamelCase_     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']					)
						else:
							_lowercase					:       Optional[int]       =		(
							    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
							)
							_lowercase					:       Any       =		(
							    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
							)
						with open(UpperCamelCase_     ,  'w'     ,  encoding='utf-8'					) as writer:
							for token_index, token in self.ids_to_tokens.items():
								if index != token_index:
									logger.warning(
									    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
									    ' Please check that the vocabulary is not corrupted!'					)
									_lowercase					:       Optional[int]       =		token_index
								writer.write(','.join(UpperCamelCase_					) + '\n'					)
								index += 1
						with open(UpperCamelCase_     ,  'w'     ,  encoding='utf-8'					) as writer:
							json.dump(self.emoji     ,  UpperCamelCase_					)
						return vocab_file, emoji_file






class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					def __init__(  self				:	int     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Tuple					)  ->       Any:



						'''simple docstring'''





						_lowercase					:       int       =		vocab  # same as swe
						_lowercase					:       Optional[int]       =		ids_to_tokens  # same as bpe
						_lowercase					:       Optional[Any]       =		emoji
						_lowercase					:       List[str]       =		np.max([len(UpperCamelCase_					) for w in self.vocab.keys()]					)
						_lowercase					:       Union[str, Any]       =		re.compile(r'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'					)
						_lowercase					:       Any       =		re.compile(r'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'					)
						_lowercase					:       Tuple       =		re.compile(r'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'					)
						_lowercase					:       Tuple       =		re.compile(
						    r'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'					)
						_lowercase					:       List[str]       =		re.compile(
						    r'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'					)
						_lowercase					:       Union[str, Any]       =		re.compile(
						    r'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'					)
						_lowercase					:       List[Any]       =		'─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
						_lowercase					:       List[str]       =		'▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
						_lowercase					:       Dict       =		str.maketrans({k: '<BLOCK>' for k in keisen + blocks}					)




					def __len__(  self				:	str					)  ->       str:



						'''simple docstring'''





						return len(self.ids_to_tokens					)




					def 						__UpperCAmelCase     (  self				:	List[Any]     ,  UpperCamelCase_				:	List[Any]					)  ->       List[str]:



						'''simple docstring'''





						_lowercase					:       int       =		self.content_repattera.sub('<URL>'     ,  UpperCamelCase_					)
						_lowercase					:       int       =		self.content_repattera.sub('<EMAIL>'     ,  UpperCamelCase_					)
						_lowercase					:       int       =		self.content_repattera.sub('<TEL>'     ,  UpperCamelCase_					)
						_lowercase					:       Optional[int]       =		self.content_repattera.sub('<DATE>'     ,  UpperCamelCase_					)
						_lowercase					:       str       =		self.content_repattera.sub('<DATE>'     ,  UpperCamelCase_					)
						_lowercase					:       Any       =		self.content_repattera.sub('<PRICE>'     ,  UpperCamelCase_					)
						_lowercase					:       int       =		content.translate(self.content_transa					)
						while "<BLOCK><BLOCK>" in content:
							_lowercase					:       Dict       =		content.replace('<BLOCK><BLOCK>'     ,  '<BLOCK>'					)
						return content




					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	Tuple     ,  UpperCamelCase_				:	Any=False					)  ->       List[str]:



						'''simple docstring'''





						_lowercase					:       Dict       =		text.replace(' '     ,  '<SP>'					)
						_lowercase					:       str       =		text.replace('　'     ,  '<SP>'					)
						_lowercase					:       Optional[Any]       =		text.replace('\r\n'     ,  '<BR>'					)
						_lowercase					:       Any       =		text.replace('\n'     ,  '<BR>'					)
						_lowercase					:       Optional[int]       =		text.replace('\r'     ,  '<BR>'					)
						_lowercase					:       Dict       =		text.replace('\t'     ,  '<TAB>'					)
						_lowercase					:       Union[str, Any]       =		text.replace('—'     ,  'ー'					)
						_lowercase					:       List[str]       =		text.replace('−'     ,  'ー'					)
						for k, v in self.emoji["emoji"].items():
							if k in text:
								_lowercase					:       List[str]       =		text.replace(UpperCamelCase_     ,  UpperCamelCase_					)
						if clean:
							_lowercase					:       Tuple       =		self.clean_text(UpperCamelCase_					)

						def check_simbol(UpperCamelCase_				:	Dict					):
							_lowercase					:       Optional[Any]       =		x.encode()
							if len(UpperCamelCase_					) == 1 and len(UpperCamelCase_					) == 2:
								_lowercase					:       List[str]       =		(int(e[0]					) << 8) + int(e[1]					)
								if (
								    (c >= 0XC_2A1 and c <= 0XC_2BF)
								    or (c >= 0XC_780 and c <= 0XC_783)
								    or (c >= 0XC_AB9 and c <= 0XC_BBF)
								    or (c >= 0XC_C80 and c <= 0XC_DA2)
								):
									return True
							return False

						def checkuae(UpperCamelCase_				:	Any					):
							_lowercase					:       Optional[int]       =		x.encode()
							if len(UpperCamelCase_					) == 1 and len(UpperCamelCase_					) == 3:
								_lowercase					:       List[str]       =		(int(e[0]					) << 16) + (int(e[1]					) << 8) + int(e[2]					)
								if c >= 0XE28_080 and c <= 0XE2B_07F:
									return True
							return False

						_lowercase					:       Dict       =		0
						_lowercase					:       Optional[Any]       =		[]
						while pos < len(UpperCamelCase_					):
							_lowercase					:       Dict       =		min(len(UpperCamelCase_					)     ,  pos + self.maxlen + 1					) if text[pos] == '<' else pos + 3
							_lowercase					:       str       =		[]  # (token_id, token, pos)
							for e in range(UpperCamelCase_     ,  UpperCamelCase_     ,  -1					):
								_lowercase					:       Optional[int]       =		text[pos:e]
								if wd in self.vocab:
									if wd[0] == "<" and len(UpperCamelCase_					) > 2:
										_lowercase					:       Optional[Any]       =		[(self.vocab[wd], wd, e)]
										break
									else:
										candidates.append((self.vocab[wd], wd, e)					)
							if len(UpperCamelCase_					) > 0:
								# the smallest token_id is adopted
								_lowercase  ,    _lowercase  ,    _lowercase					:       Optional[int]       =		sorted(UpperCamelCase_     ,  key=lambda UpperCamelCase_					: x[0]					)[0]
								result.append(UpperCamelCase_					)
								_lowercase					:       Any       =		e
							else:
								_lowercase					:       str       =		pos + 1
								_lowercase					:       Optional[Any]       =		text[pos:end]
								if check_simbol(UpperCamelCase_					):
									result.append('<KIGOU>'					)
								elif checkuae(UpperCamelCase_					):
									result.append('<U2000U2BFF>'					)
								else:
									for i in wd.encode('utf-8'					):
										result.append('<|byte%d|>' % i					)
								_lowercase					:       Optional[int]       =		end
						return result




					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	int     ,  UpperCamelCase_				:	List[str]="\n"					)  ->       List[str]:



						'''simple docstring'''





						_lowercase					:       str       =		[]
						_lowercase					:       Tuple       =		[]
						_lowercase					:       Union[str, Any]       =		self.ids_to_tokens[index][0]
						if word[:6] == "<|byte" and word[-2:] == "|>":
							byte_tokens.append(int(word[6:-2]					)					)
						else:
							if len(UpperCamelCase_					) > 0:
								words.append(bytearray(UpperCamelCase_					).decode('utf-8'     ,  errors='replace'					)					)
								_lowercase					:       Union[str, Any]       =		[]
							if word[:7] == "<|emoji" and word[-2:] == "|>":
								words.append(self.emoji['emoji_inv'][word]					)
							elif word == "<SP>":
								words.append(' '					)
							elif word == "<BR>":
								words.append(UpperCamelCase_					)
							elif word == "<TAB>":
								words.append('\t'					)
							elif word == "<BLOCK>":
								words.append('▀'					)
							elif word == "<KIGOU>":
								words.append('ǀ'					)
							elif word == "<U2000U2BFF>":
								words.append('‖'					)
							else:
								words.append(UpperCamelCase_					)
						if len(UpperCamelCase_					) > 0:
							words.append(bytearray(UpperCamelCase_					).decode('utf-8'     ,  errors='replace'					)					)
						_lowercase					:       Any       =		''.join(UpperCamelCase_					)
						return text





'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A				:    Dict     						=logging.get_logger(__name__)

_A				:    Dict     						={
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}



class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					A_						    =						"""megatron-bert"""




					def __init__(  self				:	int     ,  UpperCamelCase_				:	int=2_9056     ,  UpperCamelCase_				:	Optional[int]=1024     ,  UpperCamelCase_				:	Optional[Any]=24     ,  UpperCamelCase_				:	List[Any]=16     ,  UpperCamelCase_				:	Optional[int]=4096     ,  UpperCamelCase_				:	Optional[Any]="gelu"     ,  UpperCamelCase_				:	List[Any]=0.1     ,  UpperCamelCase_				:	Union[str, Any]=0.1     ,  UpperCamelCase_				:	int=512     ,  UpperCamelCase_				:	Dict=2     ,  UpperCamelCase_				:	Union[str, Any]=0.02     ,  UpperCamelCase_				:	Any=1E-12     ,  UpperCamelCase_				:	Tuple=0     ,  UpperCamelCase_				:	Optional[int]="absolute"     ,  UpperCamelCase_				:	Optional[Any]=True     ,  **UpperCamelCase_				:	Any     ,  )  ->       List[Any]:



						'''simple docstring'''





						super().__init__(pad_token_id=UpperCamelCase_     ,  **UpperCamelCase_					)

						_lowercase					:       Dict       =		vocab_size
						_lowercase					:       Any       =		hidden_size
						_lowercase					:       Union[str, Any]       =		num_hidden_layers
						_lowercase					:       Dict       =		num_attention_heads
						_lowercase					:       Dict       =		hidden_act
						_lowercase					:       Optional[Any]       =		intermediate_size
						_lowercase					:       Optional[int]       =		hidden_dropout_prob
						_lowercase					:       Optional[Any]       =		attention_probs_dropout_prob
						_lowercase					:       Any       =		max_position_embeddings
						_lowercase					:       str       =		type_vocab_size
						_lowercase					:       Optional[Any]       =		initializer_range
						_lowercase					:       List[str]       =		layer_norm_eps
						_lowercase					:       List[Any]       =		position_embedding_type
						_lowercase					:       Optional[Any]       =		use_cache
