import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
						import torch


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


A_      =      logging.get_logger(__name__)
A_      =      {"vocab_file": "spiece.model"}

A_      =      {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

A_      =      {
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}






class      __lowercase					(    _A    ):
						lowercase					 =     VOCAB_FILES_NAMES
						lowercase					 =     PRETRAINED_VOCAB_FILES_MAP
						lowercase					 =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowercase					 =     ['input_ids', 'attention_mask']

						def __init__(	self				:      Dict							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      Optional[Any]=False							,							__lowerCamelCase				:      List[str]=False							,							__lowerCamelCase				:      List[Any]=False							,							__lowerCamelCase				:      Optional[int]=None							,							__lowerCamelCase				:      str=None							,							__lowerCamelCase				:      Optional[int]=None							,							__lowerCamelCase				:      List[Any]=None							,							__lowerCamelCase				:      Optional[Dict[str, Any]] = None							,							**__lowerCamelCase				:      Dict							,							)							->		None:


									'''simple docstring'''

									lowercase							=		{} if sp_model_kwargs is None else sp_model_kwargs

									lowercase							=		kwargs.get('''name_or_path'''   )
									if name_or_path is None:
												logger.warning(
												    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
												    ''' you are testing the model, this can safely be ignored'''   )
												lowercase							=		'''None'''

									# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
									lowercase							=		'''<|endoftext|>''' if eos_token is None else eos_token
									lowercase							=		'''<unk>''' if unk_token is None else unk_token
									if "gpt-sw3-7b" in name_or_path:
												lowercase							=		unk_token if pad_token is None else pad_token
												lowercase							=		eos_token if bos_token is None else bos_token
									else:
												lowercase							=		'''<pad>''' if pad_token is None else pad_token
												lowercase							=		'''<s>''' if bos_token is None else bos_token

									super().__init__(
									    do_lower_case=__lowerCamelCase							,							remove_space=__lowerCamelCase							,							keep_accents=__lowerCamelCase							,							bos_token=__lowerCamelCase							,							eos_token=__lowerCamelCase							,							unk_token=__lowerCamelCase							,							pad_token=__lowerCamelCase							,							sp_model_kwargs=self.sp_model_kwargs							,							**__lowerCamelCase							,							)

									lowercase							=		do_lower_case
									lowercase							=		remove_space
									lowercase							=		keep_accents
									lowercase							=		vocab_file

									lowercase							=		spm.SentencePieceProcessor(**self.sp_model_kwargs   )
									self.sp_model.Load(__lowerCamelCase   )

									# Used for whitespace normalization in input texts
									# fmt : off
									lowercase							=		{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
									# fmt : on

									# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
									lowercase							=		re.compile(
									    f'[{"".join(map(__lowerCamelCase							,							list(range(0							,							9   )   ) + list(range(11							,							32   )   ) + list(range(1_27							,							1_60   )   ) + [1_60, 1_73, 82_03]   )   )}]'   )

						def __getstate__(	self				:      str   )							->		str:


									'''simple docstring'''

									lowercase							=		self.__dict__.copy()
									lowercase							=		None
									return state

						def __setstate__(	self				:      int							,							__lowerCamelCase				:      Any   )							->		Any:


									'''simple docstring'''

									lowercase							=		d

									# for backward compatibility
									if not hasattr(self							,							'''sp_model_kwargs'''   ):
												lowercase							=		{}

									lowercase							=		spm.SentencePieceProcessor(**self.sp_model_kwargs   )
									self.sp_model.Load(self.vocab_file   )

						@property
						# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
						def        __a		(	self				:      List[Any]   )							->		int:


									'''simple docstring'''

									return len(self.sp_model   )

						def        __a		(	self				:      List[Any]							,							__lowerCamelCase				:      str   )							->		str:


									'''simple docstring'''

									lowercase							=		self.non_printing_characters_re.sub(''''''							,							__lowerCamelCase   )

									# Normalize whitespaces
									lowercase							=		''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]   )

									# NFC Unicode normalization
									lowercase							=		unicodedata.normalize('''NFC'''							,							__lowerCamelCase   )
									return text

						def        __a		(	self				:      List[str]							,							__lowerCamelCase				:      str							,							**__lowerCamelCase				:      Any   )							->		List[str]:


									'''simple docstring'''

									lowercase							=		self.preprocess_text(__lowerCamelCase   )
									return self.sp_model.encode(__lowerCamelCase							,							out_type=__lowerCamelCase   )

						def        __a		(	self				:      List[Any]							,							__lowerCamelCase				:      str   )							->		int:


									'''simple docstring'''

									return self.sp_model.PieceToId(__lowerCamelCase   )

						def        __a		(	self				:      int							,							__lowerCamelCase				:      int   )							->		str:


									'''simple docstring'''

									return self.sp_model.IdToPiece(__lowerCamelCase   )

						@staticmethod
						def        __a		(	__lowerCamelCase				:      str   )							->		str:


									'''simple docstring'''

									return out_string

						def        __a		(	self				:      Any							,							__lowerCamelCase				:      List[str]   )							->		str:


									'''simple docstring'''

									lowercase							=		[]
									lowercase							=		''''''
									lowercase							=		False
									for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
															# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
															if not prev_is_special:
																		out_string += " "

															out_string += self.sp_model.decode(__lowerCamelCase   ) + token
															lowercase							=		True
															lowercase							=		[]
												else:
															current_sub_tokens.append(__lowerCamelCase   )
															lowercase							=		False
									out_string += self.sp_model.decode(__lowerCamelCase   )

									return out_string

						def        __a		(	self				:      Optional[int]   )							->		Dict[str, int]:


									'''simple docstring'''

									lowercase							=		{self.convert_ids_to_tokens(__lowerCamelCase   ): i for i in range(self.vocab_size   )}
									vocab.update(self.added_tokens_encoder   )
									return vocab

						def        __a		(	self				:      str							,							__lowerCamelCase				:      str							,							__lowerCamelCase				:      Optional[str] = None   )							->		Tuple[str]:


									'''simple docstring'''

									if not os.path.isdir(__lowerCamelCase   ):
												logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
												return
									lowercase							=		os.path.join(
									    __lowerCamelCase							,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

									if os.path.abspath(self.vocab_file   ) != os.path.abspath(__lowerCamelCase   ) and os.path.isfile(self.vocab_file   ):
												copyfile(self.vocab_file							,							__lowerCamelCase   )
									elif not os.path.isfile(self.vocab_file   ):
												with open(__lowerCamelCase							,							'''wb'''   ) as fi:
															lowercase							=		self.sp_model.serialized_model_proto()
															fi.write(__lowerCamelCase   )

									return (out_vocab_file,)

						def        __a		(	self				:      Tuple							,							__lowerCamelCase				:      Union[str, List[str]]							,							__lowerCamelCase				:      Union[str, bool] = False   )							->		Union[List[int], List[List[int]], "torch.Tensor"]:


									'''simple docstring'''

									if isinstance(__lowerCamelCase							,							__lowerCamelCase   ):
												lowercase							=		self.preprocess_text(__lowerCamelCase   )
												lowercase							=		self.sp_model.encode(__lowerCamelCase   )
									else:
												lowercase							=		[self.preprocess_text(__lowerCamelCase   ) for t in text]
												lowercase							=		self.sp_model.encode(__lowerCamelCase   )

									if return_tensors is True or return_tensors == "pt":
												lowercase							=		torch.tensor(__lowerCamelCase   )

									return token_ids

						def        __a		(	self				:      Tuple							,							__lowerCamelCase				:      Union[int, List[int]]   )							->		str:


									'''simple docstring'''

									return self.sp_model.decode(__lowerCamelCase   )



						def        __a		(	self				:      Any							,							__lowerCamelCase				:      "Conversation"   )							->		List[int]:


									'''simple docstring'''

									lowercase							=		[f'User: {text}' if is_user else f'Bot: {text}' for is_user, text in conversation.iter_texts()]
									lowercase							=		(
									    f'{self.eos_token}{self.bos_token}' + f'{self.bos_token}'.join(__lowerCamelCase   ) + f'{self.bos_token}Bot:'
									)
									return self.encode(text=__lowerCamelCase   )






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


A_      =      {
    "configuration_gpt_bigcode": ["GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP", "GPTBigCodeConfig"],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A_      =      [
						    "GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "GPTBigCodeForSequenceClassification",
						    "GPTBigCodeForTokenClassification",
						    "GPTBigCodeForCausalLM",
						    "GPTBigCodeModel",
						    "GPTBigCodePreTrainedModel",
						]

if TYPE_CHECKING:
						from .configuration_gpt_bigcode import GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTBigCodeConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_gpt_bigcode import (
												    GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST,
												    GPTBigCodeForCausalLM,
												    GPTBigCodeForSequenceClassification,
												    GPTBigCodeForTokenClassification,
												    GPTBigCodeModel,
												    GPTBigCodePreTrainedModel,
												)


else:
						import sys

						A_      =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





