




"""simple docstring"""


import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
     import torch


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


A_      			=       logging.get_logger(__name__)
A_      			=       {'''vocab_file''': '''spiece.model'''}

A_      			=       {
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

A_      			=       {
    '''AI-Sweden/gpt-sw3-126m''': 20_48,
    '''AI-Sweden/gpt-sw3-350m''': 20_48,
    '''AI-Sweden/gpt-sw3-1.6b''': 20_48,
    '''AI-Sweden/gpt-sw3-6.7b''': 20_48,
    '''AI-Sweden/gpt-sw3-20b''': 20_48,
}






class       lowercase(  __a				):







 '''simple docstring'''





 lowercase__				      =      VOCAB_FILES_NAMES
 lowercase__				      =      PRETRAINED_VOCAB_FILES_MAP
 lowercase__				      =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase__				      =      ["input_ids", "attention_mask"]


 def __init__(       self:	Dict,   a_:	str,   a_:	Union[str, Any]=False,   a_:	Any=False,   a_:	Tuple=False,   a_:	Tuple=None,   a_:	str=None,   a_:	List[Any]=None,   a_:	Optional[Any]=None,   a_:	Optional[Dict[str, Any]] = None,   **a_:	Union[str, Any],   ):




   '''simple docstring'''



   _snake_case							:	Union[str, Any]			=						{} if sp_model_kwargs is None else sp_model_kwargs

   _snake_case							:	Tuple			=						kwargs.get("""name_or_path"""	)
   if name_or_path is None:
     logger.warning(
         """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
         """ you are testing the model, this can safely be ignored"""	)
     _snake_case							:	str			=						"""None"""

   # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
   _snake_case							:	Tuple			=						"""<|endoftext|>""" if eos_token is None else eos_token
   _snake_case							:	str			=						"""<unk>""" if unk_token is None else unk_token
   if "gpt-sw3-7b" in name_or_path:
     _snake_case							:	List[Any]			=						unk_token if pad_token is None else pad_token
     _snake_case							:	Union[str, Any]			=						eos_token if bos_token is None else bos_token
   else:
     _snake_case							:	str			=						"""<pad>""" if pad_token is None else pad_token
     _snake_case							:	List[str]			=						"""<s>""" if bos_token is None else bos_token

   super().__init__(
       do_lower_case=a_,   remove_space=a_,   keep_accents=a_,   bos_token=a_,   eos_token=a_,   unk_token=a_,   pad_token=a_,   sp_model_kwargs=self.sp_model_kwargs,   **a_,   )

   _snake_case							:	Dict			=						do_lower_case
   _snake_case							:	Any			=						remove_space
   _snake_case							:	Optional[int]			=						keep_accents
   _snake_case							:	Optional[int]			=						vocab_file

   _snake_case							:	Dict			=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
   self.sp_model.Load(a_	)

   # Used for whitespace normalization in input texts
   # fmt : off
   _snake_case							:	str			=						{""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
   # fmt : on

   # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
   _snake_case							:	Union[str, Any]			=						re.compile(
       f"[{''.join(map(a_,   list(range(0,   9	)	) + list(range(11,   32	)	) + list(range(127,   160	)	) + [160, 173, 8_203]	)	)}]"	)


 def __getstate__(       self:	Any	):




   '''simple docstring'''



   _snake_case							:	Tuple			=						self.__dict__.copy()
   _snake_case							:	int			=						None
   return state


 def __setstate__(       self:	Any,   a_:	Optional[int]	):




   '''simple docstring'''



   _snake_case							:	Optional[Any]			=						d

   # for backward compatibility
   if not hasattr(self,   """sp_model_kwargs"""	):
     _snake_case							:	Union[str, Any]			=						{}

   _snake_case							:	Union[str, Any]			=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
   self.sp_model.Load(self.vocab_file	)


 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def 				UpperCamelCase_     (       self:	List[str]	):




   '''simple docstring'''



   return len(self.sp_model	)


 def 				UpperCamelCase_     (       self:	Dict,   a_:	str	):




   '''simple docstring'''



   _snake_case							:	Dict			=						self.non_printing_characters_re.sub("""""",   a_	)

   # Normalize whitespaces
   _snake_case							:	Any			=						"""""".join([char if char not in self.whitespaces else """ """ for char in text]	)

   # NFC Unicode normalization
   _snake_case							:	Optional[Any]			=						unicodedata.normalize("""NFC""",   a_	)
   return text


 def 				UpperCamelCase_     (       self:	str,   a_:	str,   **a_:	Union[str, Any]	):




   '''simple docstring'''



   _snake_case							:	Any			=						self.preprocess_text(a_	)
   return self.sp_model.encode(a_,   out_type=a_	)


 def 				UpperCamelCase_     (       self:	List[Any],   a_:	str	):




   '''simple docstring'''



   return self.sp_model.PieceToId(a_	)


 def 				UpperCamelCase_     (       self:	Tuple,   a_:	int	):




   '''simple docstring'''



   return self.sp_model.IdToPiece(a_	)


 @staticmethod
 def 				UpperCamelCase_     (       a_:	str	):




   '''simple docstring'''



   return out_string


 def 				UpperCamelCase_     (       self:	Tuple,   a_:	List[str]	):




   '''simple docstring'''



   _snake_case							:	List[str]			=						[]
   _snake_case							:	Optional[Any]			=						""""""
   _snake_case							:	Optional[int]			=						False
   for token in tokens:
     # make sure that special tokens are not decoded using sentencepiece model
     if token in self.all_special_tokens:
       # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
       if not prev_is_special:
         out_string += " "

       out_string += self.sp_model.decode(a_	) + token
       _snake_case							:	int			=						True
       _snake_case							:	int			=						[]
     else:
       current_sub_tokens.append(a_	)
       _snake_case							:	Union[str, Any]			=						False
   out_string += self.sp_model.decode(a_	)

   return out_string


 def 				UpperCamelCase_     (       self:	Union[str, Any]	):




   '''simple docstring'''



   _snake_case							:	str			=						{self.convert_ids_to_tokens(a_	): i for i in range(self.vocab_size	)}
   vocab.update(self.added_tokens_encoder	)
   return vocab


 def 				UpperCamelCase_     (       self:	Optional[Any],   a_:	str,   a_:	Optional[str] = None	):




   '''simple docstring'''



   if not os.path.isdir(a_	):
     logger.error(f"Vocabulary path ({save_directory}) should be a directory"	)
     return
   _snake_case							:	Optional[Any]			=						os.path.join(
       a_,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

   if os.path.abspath(self.vocab_file	) != os.path.abspath(a_	) and os.path.isfile(self.vocab_file	):
     copyfile(self.vocab_file,   a_	)
   elif not os.path.isfile(self.vocab_file	):
     with open(a_,   """wb"""	) as fi:
       _snake_case							:	Union[str, Any]			=						self.sp_model.serialized_model_proto()
       fi.write(a_	)

   return (out_vocab_file,)


 def 				UpperCamelCase_     (       self:	Any,   a_:	Union[str, List[str]],   a_:	Union[str, bool] = False	):




   '''simple docstring'''



   if isinstance(a_,   a_	):
     _snake_case							:	str			=						self.preprocess_text(a_	)
     _snake_case							:	Optional[Any]			=						self.sp_model.encode(a_	)
   else:
     _snake_case							:	int			=						[self.preprocess_text(a_	) for t in text]
     _snake_case							:	List[Any]			=						self.sp_model.encode(a_	)

   if return_tensors is True or return_tensors == "pt":
     _snake_case							:	List[str]			=						torch.tensor(a_	)

   return token_ids


 def 				UpperCamelCase_     (       self:	Optional[int],   a_:	Union[int, List[int]]	):




   '''simple docstring'''



   return self.sp_model.decode(a_	)


 def 				UpperCamelCase_     (       self:	Tuple,   a_:	"Conversation"	):




   '''simple docstring'''



   _snake_case							:	Union[str, Any]			=						[f"User: {text}" if is_user else f"Bot: {text}" for is_user, text in conversation.iter_texts()]
   _snake_case							:	List[str]			=						(
       f"{self.eos_token}{self.bos_token}" + f"{self.bos_token}".join(a_	) + f"{self.bos_token}Bot:"
   )
   return self.encode(text=a_	)








"""simple docstring"""


def       UpperCAmelCase__ (snake_case__							:      int					):
  """simple docstring"""

  if not isinstance(snake_case__	,     snake_case__					) or number < 0:
    raise ValueError("""Input must be a non-negative integer"""					)

  _snake_case							:	Dict			=						0
  while number:
    # This way we arrive at next set bit (next 1) instead of looping
    # through each bit and checking for 1s hence the
    # loop won't run 32 times it will only run the number of `1` times
    number &= number - 1
    count += 1
  return count


if __name__ == "__main__":
     import doctest

     doctest.testmod()


