



"""simple docstring"""





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
					import torch


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


__lowerCAmelCase       :					Optional[Any]      					=					logging.get_logger(__name__)
__lowerCAmelCase       :					List[Any]      					=					{'''vocab_file''': '''spiece.model'''}

__lowerCAmelCase       :					Optional[Any]      					=					{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

__lowerCAmelCase       :					Optional[Any]      					=					{
    '''AI-Sweden/gpt-sw3-126m''': 2048,
    '''AI-Sweden/gpt-sw3-350m''': 2048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2048,
    '''AI-Sweden/gpt-sw3-20b''': 2048,
}



class    _lowerCAmelCase    (				SCREAMING_SNAKE_CASE__		):





		"""simple docstring"""






		_lowerCamelCase			  =       VOCAB_FILES_NAMES
		_lowerCamelCase			  =       PRETRAINED_VOCAB_FILES_MAP
		_lowerCamelCase			  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_lowerCamelCase			  =       ['''input_ids''', '''attention_mask''']





		def __init__(					self							,    _lowercase							,    _lowercase=False							,    _lowercase=False							,    _lowercase=False							,    _lowercase=None							,    _lowercase=None							,    _lowercase=None							,    _lowercase=None							,    _lowercase = None							,    **_lowercase							,    ) ->						None:







								'''simple docstring'''

								snake_case_   : Any						=			{} if sp_model_kwargs is None else sp_model_kwargs

								snake_case_   : Optional[int]						=			kwargs.get("""name_or_path"""   )
								if name_or_path is None:
														logger.warning(
														    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
														    """ you are testing the model, this can safely be ignored"""   )
														snake_case_   : Optional[int]						=			"""None"""

								# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
								snake_case_   : Tuple						=			"""<|endoftext|>""" if eos_token is None else eos_token
								snake_case_   : str						=			"""<unk>""" if unk_token is None else unk_token
								if "gpt-sw3-7b" in name_or_path:
														snake_case_   : str						=			unk_token if pad_token is None else pad_token
														snake_case_   : Union[str, Any]						=			eos_token if bos_token is None else bos_token
								else:
														snake_case_   : Tuple						=			"""<pad>""" if pad_token is None else pad_token
														snake_case_   : Tuple						=			"""<s>""" if bos_token is None else bos_token

								super().__init__(
								    do_lower_case=_lowercase							,    remove_space=_lowercase							,    keep_accents=_lowercase							,    bos_token=_lowercase							,    eos_token=_lowercase							,    unk_token=_lowercase							,    pad_token=_lowercase							,    sp_model_kwargs=self.sp_model_kwargs							,    **_lowercase							,    )

								snake_case_   : List[str]						=			do_lower_case
								snake_case_   : int						=			remove_space
								snake_case_   : str						=			keep_accents
								snake_case_   : List[str]						=			vocab_file

								snake_case_   : Optional[Any]						=			spm.SentencePieceProcessor(**self.sp_model_kwargs   )
								self.sp_model.Load(_lowercase   )

								# Used for whitespace normalization in input texts
								# fmt : off
								snake_case_   : int						=			{""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
								# fmt : on

								# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
								snake_case_   : Dict						=			re.compile(
								    f'[{"".join(map(_lowercase							,    list(range(0							,    9   )   ) + list(range(1_1							,    3_2   )   ) + list(range(1_2_7							,    1_6_0   )   ) + [1_6_0, 1_7_3, 8_2_0_3]   )   )}]'   )





		def __getstate__(					self   ) ->						Dict:







								'''simple docstring'''

								snake_case_   : Union[str, Any]						=			self.__dict__.copy()
								snake_case_   : List[str]						=			None
								return state





		def __setstate__(					self							,    _lowercase   ) ->						Optional[int]:







								'''simple docstring'''

								snake_case_   : int						=			d

								# for backward compatibility
								if not hasattr(self							,    """sp_model_kwargs"""   ):
														snake_case_   : Any						=			{}

								snake_case_   : Tuple						=			spm.SentencePieceProcessor(**self.sp_model_kwargs   )
								self.sp_model.Load(self.vocab_file   )





		@property
		# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
		def    UpperCAmelCase__							(					self   ) ->						int:







								'''simple docstring'''

								return len(self.sp_model   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						str:







								'''simple docstring'''

								snake_case_   : str						=			self.non_printing_characters_re.sub(""""""							,    _lowercase   )

								# Normalize whitespaces
								snake_case_   : Any						=			"""""".join([char if char not in self.whitespaces else """ """ for char in text]   )

								# NFC Unicode normalization
								snake_case_   : Optional[int]						=			unicodedata.normalize("""NFC"""							,    _lowercase   )
								return text





		def    UpperCAmelCase__							(					self							,    _lowercase							,    **_lowercase   ) ->						List[str]:







								'''simple docstring'''

								snake_case_   : Optional[Any]						=			self.preprocess_text(_lowercase   )
								return self.sp_model.encode(_lowercase							,    out_type=_lowercase   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						int:







								'''simple docstring'''

								return self.sp_model.PieceToId(_lowercase   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						str:







								'''simple docstring'''

								return self.sp_model.IdToPiece(_lowercase   )





		@staticmethod
		def    UpperCAmelCase__							(					_lowercase   ) ->						str:







								'''simple docstring'''

								return out_string





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						str:







								'''simple docstring'''

								snake_case_   : str						=			[]
								snake_case_   : str						=			""""""
								snake_case_   : Any						=			False
								for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																				# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																				if not prev_is_special:
																										out_string += " "

																				out_string += self.sp_model.decode(_lowercase   ) + token
																				snake_case_   : List[Any]						=			True
																				snake_case_   : str						=			[]
														else:
																				current_sub_tokens.append(_lowercase   )
																				snake_case_   : List[Any]						=			False
								out_string += self.sp_model.decode(_lowercase   )

								return out_string





		def    UpperCAmelCase__							(					self   ) ->						Dict[str, int]:







								'''simple docstring'''

								snake_case_   : Dict						=			{self.convert_ids_to_tokens(_lowercase   ): i for i in range(self.vocab_size   )}
								vocab.update(self.added_tokens_encoder   )
								return vocab





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase = None   ) ->						Tuple[str]:







								'''simple docstring'''

								if not os.path.isdir(_lowercase   ):
														logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
														return
								snake_case_   : List[Any]						=			os.path.join(
								    _lowercase							,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )

								if os.path.abspath(self.vocab_file   ) != os.path.abspath(_lowercase   ) and os.path.isfile(self.vocab_file   ):
														copyfile(self.vocab_file							,    _lowercase   )
								elif not os.path.isfile(self.vocab_file   ):
														with open(_lowercase							,    """wb"""   ) as fi:
																				snake_case_   : Union[str, Any]						=			self.sp_model.serialized_model_proto()
																				fi.write(_lowercase   )

								return (out_vocab_file,)





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase = False   ) ->						Union[List[int], List[List[int]], "torch.Tensor"]:







								'''simple docstring'''

								if isinstance(_lowercase							,    _lowercase   ):
														snake_case_   : Optional[int]						=			self.preprocess_text(_lowercase   )
														snake_case_   : int						=			self.sp_model.encode(_lowercase   )
								else:
														snake_case_   : str						=			[self.preprocess_text(_lowercase   ) for t in text]
														snake_case_   : List[Any]						=			self.sp_model.encode(_lowercase   )

								if return_tensors is True or return_tensors == "pt":
														snake_case_   : List[str]						=			torch.tensor(_lowercase   )

								return token_ids





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						str:







								'''simple docstring'''

								return self.sp_model.decode(_lowercase   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						List[int]:







								'''simple docstring'''

								snake_case_   : Optional[int]						=			[f'User: {text}' if is_user else f'Bot: {text}' for is_user, text in conversation.iter_texts()]
								snake_case_   : Dict						=			(
								    f'{self.eos_token}{self.bos_token}' + f'{self.bos_token}'.join(_lowercase   ) + f'{self.bos_token}Bot:'
								)
								return self.encode(text=_lowercase   )






"""simple docstring"""





import math

import tensorflow as tf
from packaging import version
def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[Any]    ):


						'''simple docstring'''






						snake_case_   : int						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : int						=			0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0    )					, x.dtype    )    ))

						return x * cdf
def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[str]    ):


						'''simple docstring'''






						snake_case_   : int						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : List[Any]						=			tf.cast(math.pi					, x.dtype    )
						snake_case_   : int						=			tf.cast(0.044_715					, x.dtype    )
						snake_case_   : Optional[int]						=			0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi    ) * (x + coeff * tf.pow(__UpperCamelCase					, 3    ))    ))

						return x * cdf
def 	__lowerCAmelCase						(		__UpperCamelCase      :			str    ):


						'''simple docstring'''






						snake_case_   : Optional[Any]						=			tf.convert_to_tensor(__UpperCamelCase    )

						return x * tf.tanh(tf.math.softplus(__UpperCamelCase    )    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Any    ):


						'''simple docstring'''






						snake_case_   : str						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : int						=			tf.cast(0.044_715					, x.dtype    )
						snake_case_   : Optional[int]						=			tf.cast(0.7_978_845_608					, x.dtype    )

						return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)    ))
def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[str]    ):


						'''simple docstring'''






						snake_case_   : Tuple						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : str						=			tf.cast(1.702					, x.dtype    )
						return x * tf.math.sigmoid(coeff * x    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int    ):


						'''simple docstring'''






						return tf.clip_by_value(_gelu(__UpperCamelCase    )					, -1_0					, 1_0    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Optional[int]					, __UpperCamelCase      :			List[str]=-1    ):


						'''simple docstring'''






						snake_case_			,		snake_case_   : List[Any]						=			tf.split(__UpperCamelCase					, 2					, axis=__UpperCamelCase    )
						return a * tf.math.sigmoid(__UpperCamelCase    )


if version.parse(tf.version.VERSION) >= version.parse('''2.4'''):
					def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[Any]    ):


											'''simple docstring'''






											return tf.keras.activations.gelu(__UpperCamelCase					, approximate=__UpperCamelCase    )

					__lowerCAmelCase       :					int      					=					tf.keras.activations.gelu






					__lowerCAmelCase       :					Optional[Any]      					=					approximate_gelu_wrap
else:
					__lowerCAmelCase       :					List[Any]      					=					_gelu
					__lowerCAmelCase       :					Any      					=					_gelu_new


__lowerCAmelCase       :					Dict      					=					{
    '''gelu''': gelu,
    '''gelu_10''': gelu_aa,
    '''gelu_fast''': gelu_fast,
    '''gelu_new''': gelu_new,
    '''glu''': glu,
    '''mish''': mish,
    '''quick_gelu''': quick_gelu,
    '''relu''': tf.keras.activations.relu,
    '''sigmoid''': tf.keras.activations.sigmoid,
    '''silu''': tf.keras.activations.swish,
    '''swish''': tf.keras.activations.swish,
    '''tanh''': tf.keras.activations.tanh,
}






def 	__lowerCAmelCase						(		__UpperCamelCase      :			Any    ):


						'''simple docstring'''






						if activation_string in ACTaFN:
												return ACTaFN[activation_string]
						else:
												raise KeyError(F'function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()    )}'    )

