



"""simple docstring"""





import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase       :					Optional[int]      					=					logging.get_logger(__name__)

__lowerCAmelCase       :					List[str]      					=					{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__lowerCAmelCase       :					Optional[int]      					=					{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

__lowerCAmelCase       :					Optional[Any]      					=					{
    '''ctrl''': 256,
}

__lowerCAmelCase       :					str      					=					{
    '''Pregnancy''': 16_8629,
    '''Christianity''': 7675,
    '''Explain''': 10_6423,
    '''Fitness''': 6_3440,
    '''Saving''': 6_3163,
    '''Ask''': 2_7171,
    '''Ass''': 9_5985,
    '''Joke''': 16_3509,
    '''Questions''': 4_5622,
    '''Thoughts''': 4_9605,
    '''Retail''': 5_2342,
    '''Feminism''': 16_4338,
    '''Writing''': 1_1992,
    '''Atheism''': 19_2263,
    '''Netflix''': 4_8616,
    '''Computing''': 3_9639,
    '''Opinion''': 4_3213,
    '''Alone''': 4_4967,
    '''Funny''': 5_8917,
    '''Gaming''': 4_0358,
    '''Human''': 4088,
    '''India''': 1331,
    '''Joker''': 7_7138,
    '''Diet''': 3_6206,
    '''Legal''': 1_1859,
    '''Norman''': 4939,
    '''Tip''': 7_2689,
    '''Weight''': 5_2343,
    '''Movies''': 4_6273,
    '''Running''': 2_3425,
    '''Science''': 2090,
    '''Horror''': 3_7793,
    '''Confession''': 6_0572,
    '''Finance''': 1_2250,
    '''Politics''': 1_6360,
    '''Scary''': 19_1985,
    '''Support''': 1_2654,
    '''Technologies''': 3_2516,
    '''Teenage''': 6_6160,
    '''Event''': 3_2769,
    '''Learned''': 6_7460,
    '''Notion''': 18_2770,
    '''Wikipedia''': 3_7583,
    '''Books''': 6665,
    '''Extract''': 7_6050,
    '''Confessions''': 10_2701,
    '''Conspiracy''': 7_5932,
    '''Links''': 6_3674,
    '''Narcissus''': 15_0425,
    '''Relationship''': 5_4766,
    '''Relationships''': 13_4796,
    '''Reviews''': 4_1671,
    '''News''': 4256,
    '''Translation''': 2_6820,
    '''multilingual''': 12_8406,
}
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Optional[int]    ):


						'''simple docstring'''






						snake_case_   : Any						=			set()
						snake_case_   : Any						=			word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)    )
												snake_case_   : int						=			char

						snake_case_   : Optional[Any]						=			set(__UpperCamelCase    )
						return pairs






class    _lowerCAmelCase    (				SCREAMING_SNAKE_CASE__		):





		"""simple docstring"""






		_lowerCamelCase			  =       VOCAB_FILES_NAMES
		_lowerCamelCase			  =       PRETRAINED_VOCAB_FILES_MAP
		_lowerCamelCase			  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_lowerCamelCase			  =       CONTROL_CODES





		def __init__(					self							,    _lowercase							,    _lowercase							,    _lowercase="<unk>"							,    **_lowercase   ) ->						Any:







								'''simple docstring'''

								super().__init__(unk_token=_lowercase							,    **_lowercase   )

								with open(_lowercase							,    encoding="""utf-8"""   ) as vocab_handle:
														snake_case_   : List[str]						=			json.load(_lowercase   )
								snake_case_   : List[Any]						=			{v: k for k, v in self.encoder.items()}
								with open(_lowercase							,    encoding="""utf-8"""   ) as merges_handle:
														snake_case_   : List[str]						=			merges_handle.read().split("""\n"""   )[1:-1]
								snake_case_   : Any						=			[tuple(merge.split()   ) for merge in merges]
								snake_case_   : List[str]						=			dict(zip(_lowercase							,    range(len(_lowercase   )   )   )   )
								snake_case_   : Union[str, Any]						=			{}





		@property
		def    UpperCAmelCase__							(					self   ) ->						List[str]:







								'''simple docstring'''

								return len(self.encoder   )





		def    UpperCAmelCase__							(					self   ) ->						Any:







								'''simple docstring'''

								return dict(self.encoder							,    **self.added_tokens_encoder   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						Dict:







								'''simple docstring'''

								if token in self.cache:
														return self.cache[token]
								snake_case_   : Optional[int]						=			tuple(_lowercase   )
								snake_case_   : Any						=			tuple(list(word[:-1]   ) + [word[-1] + """</w>"""]   )
								snake_case_   : str						=			get_pairs(_lowercase   )

								if not pairs:
														return token

								while True:
														snake_case_   : List[Any]						=			min(_lowercase							,    key=lambda _lowercase   : self.bpe_ranks.get(_lowercase							,    float("""inf"""   )   )   )
														if bigram not in self.bpe_ranks:
																				break
														snake_case_			,		snake_case_   : str						=			bigram
														snake_case_   : Optional[Any]						=			[]
														snake_case_   : List[Any]						=			0
														while i < len(_lowercase   ):
																				try:
																										snake_case_   : str						=			word.index(_lowercase							,    _lowercase   )
																				except ValueError:
																										new_word.extend(word[i:]   )
																										break
																				else:
																										new_word.extend(word[i:j]   )
																										snake_case_   : Any						=			j

																				if word[i] == first and i < len(_lowercase   ) - 1 and word[i + 1] == second:
																										new_word.append(first + second   )
																										i += 2
																				else:
																										new_word.append(word[i]   )
																										i += 1
														snake_case_   : List[Any]						=			tuple(_lowercase   )
														snake_case_   : Optional[int]						=			new_word
														if len(_lowercase   ) == 1:
																				break
														else:
																				snake_case_   : Optional[Any]						=			get_pairs(_lowercase   )
								snake_case_   : Any						=			"""@@ """.join(_lowercase   )
								snake_case_   : Optional[Any]						=			word[:-4]
								snake_case_   : List[Any]						=			word
								return word





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						Optional[Any]:







								'''simple docstring'''

								snake_case_   : Any						=			[]

								snake_case_   : List[Any]						=			re.findall(R"""\S+\n?"""							,    _lowercase   )

								for token in words:
														split_tokens.extend(list(self.bpe(_lowercase   ).split(""" """   )   )   )
								return split_tokens





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						Optional[int]:







								'''simple docstring'''

								return self.encoder.get(_lowercase							,    self.encoder.get(self.unk_token   )   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						Any:







								'''simple docstring'''

								return self.decoder.get(_lowercase							,    self.unk_token   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						List[str]:







								'''simple docstring'''

								snake_case_   : Dict						=			""" """.join(_lowercase   ).replace("""@@ """							,    """"""   ).strip()
								return out_string





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase = None   ) ->						Tuple[str]:







								'''simple docstring'''

								if not os.path.isdir(_lowercase   ):
														logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
														return
								snake_case_   : Any						=			os.path.join(
								    _lowercase							,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
								snake_case_   : Dict						=			os.path.join(
								    _lowercase							,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

								with open(_lowercase							,    """w"""							,    encoding="""utf-8"""   ) as f:
														f.write(json.dumps(self.encoder							,    indent=2							,    sort_keys=_lowercase							,    ensure_ascii=_lowercase   ) + """\n"""   )

								snake_case_   : Union[str, Any]						=			0
								with open(_lowercase							,    """w"""							,    encoding="""utf-8"""   ) as writer:
														writer.write("""#version: 0.2\n"""   )
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,    key=lambda _lowercase   : kv[1]   ):
																				if index != token_index:
																										logger.warning(
																										    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																										    """ Please check that the tokenizer is not corrupted!"""   )
																										snake_case_   : Dict						=			token_index
																				writer.write(""" """.join(_lowercase   ) + """\n"""   )
																				index += 1

								return vocab_file, merge_file

		# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
		#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
		#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
		#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
		#     return ''.join(tokens_generated_so_far)






"""simple docstring"""





import math

import tensorflow as tf
from packaging import version
def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[Any]    ):


						'''simple docstring'''






						snake_case_   : int						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : int						=			0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0    )					, x.dtype    )    ))

						return x * cdf
def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[str]    ):


						'''simple docstring'''






						snake_case_   : int						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : List[Any]						=			tf.cast(math.pi					, x.dtype    )
						snake_case_   : int						=			tf.cast(0.044_715					, x.dtype    )
						snake_case_   : Optional[int]						=			0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi    ) * (x + coeff * tf.pow(__UpperCamelCase					, 3    ))    ))

						return x * cdf
def 	__lowerCAmelCase						(		__UpperCamelCase      :			str    ):


						'''simple docstring'''






						snake_case_   : Optional[Any]						=			tf.convert_to_tensor(__UpperCamelCase    )

						return x * tf.tanh(tf.math.softplus(__UpperCamelCase    )    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Any    ):


						'''simple docstring'''






						snake_case_   : str						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : int						=			tf.cast(0.044_715					, x.dtype    )
						snake_case_   : Optional[int]						=			tf.cast(0.7_978_845_608					, x.dtype    )

						return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)    ))
def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[str]    ):


						'''simple docstring'''






						snake_case_   : Tuple						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : str						=			tf.cast(1.702					, x.dtype    )
						return x * tf.math.sigmoid(coeff * x    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int    ):


						'''simple docstring'''






						return tf.clip_by_value(_gelu(__UpperCamelCase    )					, -1_0					, 1_0    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Optional[int]					, __UpperCamelCase      :			List[str]=-1    ):


						'''simple docstring'''






						snake_case_			,		snake_case_   : List[Any]						=			tf.split(__UpperCamelCase					, 2					, axis=__UpperCamelCase    )
						return a * tf.math.sigmoid(__UpperCamelCase    )


if version.parse(tf.version.VERSION) >= version.parse('''2.4'''):
					def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[Any]    ):


											'''simple docstring'''






											return tf.keras.activations.gelu(__UpperCamelCase					, approximate=__UpperCamelCase    )

					__lowerCAmelCase       :					int      					=					tf.keras.activations.gelu






					__lowerCAmelCase       :					Optional[Any]      					=					approximate_gelu_wrap
else:
					__lowerCAmelCase       :					List[Any]      					=					_gelu
					__lowerCAmelCase       :					Any      					=					_gelu_new


__lowerCAmelCase       :					Dict      					=					{
    '''gelu''': gelu,
    '''gelu_10''': gelu_aa,
    '''gelu_fast''': gelu_fast,
    '''gelu_new''': gelu_new,
    '''glu''': glu,
    '''mish''': mish,
    '''quick_gelu''': quick_gelu,
    '''relu''': tf.keras.activations.relu,
    '''sigmoid''': tf.keras.activations.sigmoid,
    '''silu''': tf.keras.activations.swish,
    '''swish''': tf.keras.activations.swish,
    '''tanh''': tf.keras.activations.tanh,
}






def 	__lowerCAmelCase						(		__UpperCamelCase      :			Any    ):


						'''simple docstring'''






						if activation_string in ACTaFN:
												return ACTaFN[activation_string]
						else:
												raise KeyError(F'function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()    )}'    )

