'''simple docstring'''




import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
      import torch


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation


_lowercase		:				List[str]	       =		logging.get_logger(__name__)
_lowercase		:				List[str]	       =		{"vocab_file": "spiece.model"}

_lowercase		:				List[Any]	       =		{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

_lowercase		:				Dict	       =		{
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}





class 					__magic_name__		(					_UpperCAmelCase):
   UpperCamelCase__       = VOCAB_FILES_NAMES
   UpperCamelCase__       = PRETRAINED_VOCAB_FILES_MAP
   UpperCamelCase__       = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCamelCase__       = ['''input_ids''', '''attention_mask''']







   def __init__(			self       :						str       ,    lowercase_       :						Optional[int]       ,    lowercase_       :						Any=False       ,    lowercase_       :						Union[str, Any]=False       ,    lowercase_       :						Dict=False       ,    lowercase_       :						str=None       ,    lowercase_       :						Optional[int]=None       ,    lowercase_       :						str=None       ,    lowercase_       :						Any=None       ,    lowercase_       :						Optional[Dict[str, Any]] = None       ,    **lowercase_       :						Tuple       ,    ):
          lowercase_      :    Optional[int]       		= {} if sp_model_kwargs is None else sp_model_kwargs

          lowercase_      :    Union[str, Any]       		= kwargs.get("""name_or_path"""					)
          if name_or_path is None:
                 logger.warning(
                     """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
                     """ you are testing the model, this can safely be ignored"""					)
                 lowercase_      :    List[Any]       		= """None"""

          # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
          lowercase_      :    Dict       		= """<|endoftext|>""" if eos_token is None else eos_token
          lowercase_      :    Optional[Any]       		= """<unk>""" if unk_token is None else unk_token
          if "gpt-sw3-7b" in name_or_path:
                 lowercase_      :    Union[str, Any]       		= unk_token if pad_token is None else pad_token
                 lowercase_      :    int       		= eos_token if bos_token is None else bos_token
          else:
                 lowercase_      :    Optional[Any]       		= """<pad>""" if pad_token is None else pad_token
                 lowercase_      :    int       		= """<s>""" if bos_token is None else bos_token

          super().__init__(
              do_lower_case=lowercase_       ,    remove_space=lowercase_       ,    keep_accents=lowercase_       ,    bos_token=lowercase_       ,    eos_token=lowercase_       ,    unk_token=lowercase_       ,    pad_token=lowercase_       ,    sp_model_kwargs=self.sp_model_kwargs       ,    **lowercase_       ,    )

          lowercase_      :    Optional[Any]       		= do_lower_case
          lowercase_      :    List[Any]       		= remove_space
          lowercase_      :    Tuple       		= keep_accents
          lowercase_      :    List[Any]       		= vocab_file

          lowercase_      :    Dict       		= spm.SentencePieceProcessor(**self.sp_model_kwargs					)
          self.sp_model.Load(lowercase_					)

          # Used for whitespace normalization in input texts
          # fmt : off
          lowercase_      :    Optional[Any]       		= {""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
          # fmt : on

          # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
          lowercase_      :    int       		= re.compile(
              f'''[{''.join(map(lowercase_       ,    list(range(0       ,    9					)					) + list(range(11       ,    32					)					) + list(range(127       ,    160					)					) + [160, 173, 8203]					)					)}]'''					)







   def __getstate__(			self       :						Tuple					):
          lowercase_      :    int       		= self.__dict__.copy()
          lowercase_      :    Union[str, Any]       		= None
          return state







   def __setstate__(			self       :						Union[str, Any]       ,    lowercase_       :						str					):
          lowercase_      :    Any       		= d

          # for backward compatibility
          if not hasattr(self       ,    """sp_model_kwargs"""					):
                 lowercase_      :    Optional[Any]       		= {}

          lowercase_      :    Any       		= spm.SentencePieceProcessor(**self.sp_model_kwargs					)
          self.sp_model.Load(self.vocab_file					)







   @property
   # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
   def 			SCREAMING_SNAKE_CASE_  (			self       :						Optional[int]					):
          return len(self.sp_model					)







   def 			SCREAMING_SNAKE_CASE_  (			self       :						Any       ,    lowercase_       :						str					):
          lowercase_      :    Union[str, Any]       		= self.non_printing_characters_re.sub(""""""       ,    lowercase_					)

          # Normalize whitespaces
          lowercase_      :    Union[str, Any]       		= """""".join([char if char not in self.whitespaces else """ """ for char in text]					)

          # NFC Unicode normalization
          lowercase_      :    Tuple       		= unicodedata.normalize("""NFC"""       ,    lowercase_					)
          return text







   def 			SCREAMING_SNAKE_CASE_  (			self       :						int       ,    lowercase_       :						str       ,    **lowercase_       :						List[Any]					):
          lowercase_      :    Tuple       		= self.preprocess_text(lowercase_					)
          return self.sp_model.encode(lowercase_       ,    out_type=lowercase_					)







   def 			SCREAMING_SNAKE_CASE_  (			self       :						Tuple       ,    lowercase_       :						str					):
          return self.sp_model.PieceToId(lowercase_					)







   def 			SCREAMING_SNAKE_CASE_  (			self       :						Optional[Any]       ,    lowercase_       :						int					):
          return self.sp_model.IdToPiece(lowercase_					)







   @staticmethod
   def 			SCREAMING_SNAKE_CASE_  (			lowercase_       :						str					):
          return out_string







   def 			SCREAMING_SNAKE_CASE_  (			self       :						List[Any]       ,    lowercase_       :						List[str]					):
          lowercase_      :    Optional[int]       		= []
          lowercase_      :    Optional[Any]       		= """"""
          lowercase_      :    Dict       		= False
          for token in tokens:
                 # make sure that special tokens are not decoded using sentencepiece model
                 if token in self.all_special_tokens:
                        # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                        if not prev_is_special:
                               out_string += " "

                        out_string += self.sp_model.decode(lowercase_					) + token
                        lowercase_      :    Optional[Any]       		= True
                        lowercase_      :    Optional[Any]       		= []
                 else:
                        current_sub_tokens.append(lowercase_					)
                        lowercase_      :    List[str]       		= False
          out_string += self.sp_model.decode(lowercase_					)

          return out_string







   def 			SCREAMING_SNAKE_CASE_  (			self       :						str					):
          lowercase_      :    List[Any]       		= {self.convert_ids_to_tokens(lowercase_					): i for i in range(self.vocab_size					)}
          vocab.update(self.added_tokens_encoder					)
          return vocab







   def 			SCREAMING_SNAKE_CASE_  (			self       :						Any       ,    lowercase_       :						str       ,    lowercase_       :						Optional[str] = None					):
          if not os.path.isdir(lowercase_					):
                 logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
                 return
          lowercase_      :    str       		= os.path.join(
              lowercase_       ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

          if os.path.abspath(self.vocab_file					) != os.path.abspath(lowercase_					) and os.path.isfile(self.vocab_file					):
                 copyfile(self.vocab_file       ,    lowercase_					)
          elif not os.path.isfile(self.vocab_file					):
                 with open(lowercase_       ,    """wb"""					) as fi:
                        lowercase_      :    Any       		= self.sp_model.serialized_model_proto()
                        fi.write(lowercase_					)

          return (out_vocab_file,)







   def 			SCREAMING_SNAKE_CASE_  (			self       :						Union[str, Any]       ,    lowercase_       :						Union[str, List[str]]       ,    lowercase_       :						Union[str, bool] = False					):

          if isinstance(lowercase_       ,    lowercase_					):
                 lowercase_      :    str       		= self.preprocess_text(lowercase_					)
                 lowercase_      :    Any       		= self.sp_model.encode(lowercase_					)
          else:
                 lowercase_      :    int       		= [self.preprocess_text(lowercase_					) for t in text]
                 lowercase_      :    Dict       		= self.sp_model.encode(lowercase_					)

          if return_tensors is True or return_tensors == "pt":
                 lowercase_      :    List[str]       		= torch.tensor(lowercase_					)

          return token_ids







   def 			SCREAMING_SNAKE_CASE_  (			self       :						Optional[int]       ,    lowercase_       :						Union[int, List[int]]					):
          return self.sp_model.decode(lowercase_					)







   def 			SCREAMING_SNAKE_CASE_  (			self       :						List[Any]       ,    lowercase_       :						"Conversation"					):
          lowercase_      :    Optional[Any]       		= [f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
          lowercase_      :    Optional[Any]       		= (
              f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(lowercase_					) + f'''{self.bos_token}Bot:'''
          )
          return self.encode(text=lowercase_					)






'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_lowercase		:				Dict	       =		{
    "configuration_bloom": ["BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP", "BloomConfig", "BloomOnnxConfig"],
}
try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowercase		:				Dict	       =		["BloomTokenizerFast"]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowercase		:				Dict	       =		[
						    "BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "BloomForCausalLM",
						    "BloomModel",
						    "BloomPreTrainedModel",
						    "BloomForSequenceClassification",
						    "BloomForTokenClassification",
						    "BloomForQuestionAnswering",
						]

if TYPE_CHECKING:
						from .configuration_bloom import BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP, BloomConfig, BloomOnnxConfig

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_bloom_fast import BloomTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_bloom import (
												    BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST,
												    BloomForCausalLM,
												    BloomForQuestionAnswering,
												    BloomForSequenceClassification,
												    BloomForTokenClassification,
												    BloomModel,
												    BloomPreTrainedModel,
												)

else:
						import sys

						_lowercase		:				Optional[int]	       =		_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





