

'''simple docstring'''





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
  import torch


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


__UpperCAmelCase    				=     logging.get_logger(__name__)
__UpperCAmelCase    				=     {'''vocab_file''': '''spiece.model'''}

__UpperCAmelCase    				=     {
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

__UpperCAmelCase    				=     {
    '''AI-Sweden/gpt-sw3-126m''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-350m''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-1.6b''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-6.7b''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-20b''': 2_0_4_8,
}





class A__    ( SCREAMING_SNAKE_CASE__ ):



 """simple docstring"""







 _lowercase     :					int      =						VOCAB_FILES_NAMES
 _lowercase     :					Optional[int]      =						PRETRAINED_VOCAB_FILES_MAP
 _lowercase     :					Dict      =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowercase     :					Tuple      =						['''input_ids''', '''attention_mask''']

 def __init__(		self :	Optional[int]     ,						A_ :	int     ,						A_ :	int=False     ,						A_ :	List[Any]=False     ,						A_ :	Union[str, Any]=False     ,						A_ :	Tuple=None     ,						A_ :	List[Any]=None     ,						A_ :	Dict=None     ,						A_ :	Any=None     ,						A_ :	List[str] = None     ,						**A_ :	Any     ,						):






      '''simple docstring'''




      _lowerCAmelCase   :					Any        =    {} if sp_model_kwargs is None else sp_model_kwargs

      _lowerCAmelCase   :					Optional[int]        =    kwargs.get("name_or_path"       )
      if name_or_path is None:
           logger.warning(
               "name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"
               " you are testing the model, this can safely be ignored"       )
           _lowerCAmelCase   :					Optional[int]        =    """None"""

      # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
      _lowerCAmelCase   :					Tuple        =    """<|endoftext|>""" if eos_token is None else eos_token
      _lowerCAmelCase   :					str        =    """<unk>""" if unk_token is None else unk_token
      if "gpt-sw3-7b" in name_or_path:
           _lowerCAmelCase   :					str        =    unk_token if pad_token is None else pad_token
           _lowerCAmelCase   :					Union[str, Any]        =    eos_token if bos_token is None else bos_token
      else:
           _lowerCAmelCase   :					Tuple        =    """<pad>""" if pad_token is None else pad_token
           _lowerCAmelCase   :					Tuple        =    """<s>""" if bos_token is None else bos_token

      super().__init__(
          do_lower_case=_lowercase     ,						remove_space=_lowercase     ,						keep_accents=_lowercase     ,						bos_token=_lowercase     ,						eos_token=_lowercase     ,						unk_token=_lowercase     ,						pad_token=_lowercase     ,						sp_model_kwargs=self.sp_model_kwargs     ,						**_lowercase     ,						)

      _lowerCAmelCase   :					List[str]        =    do_lower_case
      _lowerCAmelCase   :					int        =    remove_space
      _lowerCAmelCase   :					str        =    keep_accents
      _lowerCAmelCase   :					List[str]        =    vocab_file

      _lowerCAmelCase   :					Optional[Any]        =    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
      self.sp_model.Load(_lowercase       )

      # Used for whitespace normalization in input texts
      # fmt : off
      _lowerCAmelCase   :					int        =    {""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
      # fmt : on

      # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
      _lowerCAmelCase   :					Dict        =    re.compile(
          F'''[{''.join(map(_lowercase     ,						list(range(0     ,						9       )       ) + list(range(1_1     ,						3_2       )       ) + list(range(1_2_7     ,						1_6_0       )       ) + [1_6_0, 1_7_3, 8_2_0_3]       )       )}]'''       )

 def __getstate__(		self :	Union[str, Any]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Union[str, Any]        =    self.__dict__.copy()
      _lowerCAmelCase   :					List[str]        =    None
      return state

 def __setstate__(		self :	Dict     ,						A_ :	Tuple       ):






      '''simple docstring'''




      _lowerCAmelCase   :					int        =    d

      # for backward compatibility
      if not hasattr(self     ,						"sp_model_kwargs"       ):
           _lowerCAmelCase   :					Any        =    {}

      _lowerCAmelCase   :					Tuple        =    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
      self.sp_model.Load(self.vocab_file       )

 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def    __magic_name__       (		self :	Tuple       ):






      '''simple docstring'''




      return len(self.sp_model       )

 def    __magic_name__       (		self :	str     ,						A_ :	Tuple       ):






      '''simple docstring'''




      _lowerCAmelCase   :					str        =    self.non_printing_characters_re.sub(""     ,						_lowercase       )

      # Normalize whitespaces
      _lowerCAmelCase   :					Any        =    """""".join([char if char not in self.whitespaces else " " for char in text]       )

      # NFC Unicode normalization
      _lowerCAmelCase   :					Optional[int]        =    unicodedata.normalize("NFC"     ,						_lowercase       )
      return text

 def    __magic_name__       (		self :	Any     ,						A_ :	int     ,						**A_ :	Optional[Any]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Optional[Any]        =    self.preprocess_text(_lowercase       )
      return self.sp_model.encode(_lowercase     ,						out_type=_lowercase       )

 def    __magic_name__       (		self :	Union[str, Any]     ,						A_ :	Optional[int]       ):






      '''simple docstring'''




      return self.sp_model.PieceToId(_lowercase       )

 def    __magic_name__       (		self :	Optional[int]     ,						A_ :	Any       ):






      '''simple docstring'''




      return self.sp_model.IdToPiece(_lowercase       )

 @staticmethod
 def    __magic_name__       (		A_ :	Any       ):






      '''simple docstring'''




      return out_string

 def    __magic_name__       (		self :	Union[str, Any]     ,						A_ :	Tuple       ):






      '''simple docstring'''




      _lowerCAmelCase   :					str        =    []
      _lowerCAmelCase   :					str        =    """"""
      _lowerCAmelCase   :					Any        =    False
      for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
                # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                if not prev_is_special:
                     out_string += " "

                out_string += self.sp_model.decode(_lowercase       ) + token
                _lowerCAmelCase   :					List[Any]        =    True
                _lowerCAmelCase   :					str        =    []
           else:
                current_sub_tokens.append(_lowercase       )
                _lowerCAmelCase   :					List[Any]        =    False
      out_string += self.sp_model.decode(_lowercase       )

      return out_string

 def    __magic_name__       (		self :	Any       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Dict        =    {self.convert_ids_to_tokens(_lowercase       ): i for i in range(self.vocab_size       )}
      vocab.update(self.added_tokens_encoder       )
      return vocab

 def    __magic_name__       (		self :	List[str]     ,						A_ :	int     ,						A_ :	Dict = None       ):






      '''simple docstring'''




      if not os.path.isdir(_lowercase       ):
           logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
           return
      _lowerCAmelCase   :					List[Any]        =    os.path.join(
          _lowercase     ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

      if os.path.abspath(self.vocab_file       ) != os.path.abspath(_lowercase       ) and os.path.isfile(self.vocab_file       ):
           copyfile(self.vocab_file     ,						_lowercase       )
      elif not os.path.isfile(self.vocab_file       ):
           with open(_lowercase     ,						"wb"       ) as fi:
                _lowerCAmelCase   :					Union[str, Any]        =    self.sp_model.serialized_model_proto()
                fi.write(_lowercase       )

      return (out_vocab_file,)

 def    __magic_name__       (		self :	str     ,						A_ :	List[str]     ,						A_ :	Optional[Any] = False       ):






      '''simple docstring'''




      if isinstance(_lowercase     ,						_lowercase       ):
           _lowerCAmelCase   :					Optional[int]        =    self.preprocess_text(_lowercase       )
           _lowerCAmelCase   :					int        =    self.sp_model.encode(_lowercase       )
      else:
           _lowerCAmelCase   :					str        =    [self.preprocess_text(_lowercase       ) for t in text]
           _lowerCAmelCase   :					List[Any]        =    self.sp_model.encode(_lowercase       )

      if return_tensors is True or return_tensors == "pt":
           _lowerCAmelCase   :					List[str]        =    torch.tensor(_lowercase       )

      return token_ids

 def    __magic_name__       (		self :	Any     ,						A_ :	int       ):






      '''simple docstring'''




      return self.sp_model.decode(_lowercase       )


 def    __magic_name__       (		self :	Optional[Any]     ,						A_ :	List[Any]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Optional[int]        =    [F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
      _lowerCAmelCase   :					Dict        =    (
          F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(_lowercase       ) + F'''{self.bos_token}Bot:'''
      )
      return self.encode(text=_lowercase       )









from dataclasses import dataclass, field
from typing import Tuple

from ..utils import cached_property, is_tf_available, logging, requires_backends
from .benchmark_args_utils import BenchmarkArguments


if is_tf_available():
  import tensorflow as tf


__UpperCAmelCase    				=     logging.get_logger(__name__)





@dataclass
class A__    ( A ):



 """simple docstring"""







 _lowercase     :					List[Any]      =						[
     '''no_inference''',
     '''no_cuda''',
     '''no_tpu''',
     '''no_speed''',
     '''no_memory''',
     '''no_env_print''',
     '''no_multi_process''',
 ]

 def __init__(		self :	List[Any]     ,						**A_ :	str       ):






      '''simple docstring'''




      for deprecated_arg in self.deprecated_args:
           if deprecated_arg in kwargs:
                _lowerCAmelCase   :					str        =    deprecated_arg[3:]
                _lowerCAmelCase   :					List[Any]        =    not kwargs.pop(A_       )
                logger.warning(
                    F'''{deprecated_arg} is depreciated. Please use --no-{positive_arg} or'''
                    F''' {positive_arg}={kwargs[positive_arg]}'''       )
      _lowerCAmelCase   :					Any        =    kwargs.pop("tpu_name"     ,						self.tpu_name       )
      _lowerCAmelCase   :					str        =    kwargs.pop("device_idx"     ,						self.device_idx       )
      _lowerCAmelCase   :					str        =    kwargs.pop("eager_mode"     ,						self.eager_mode       )
      _lowerCAmelCase   :					Tuple        =    kwargs.pop("use_xla"     ,						self.use_xla       )
      super().__init__(**A_       )

 _lowercase     :					str      =						field(
     default=A  ,		metadata={'''help''': '''Name of TPU'''}  ,		)
 _lowercase     :					int      =						field(
     default=0  ,		metadata={'''help''': '''CPU / GPU device index. Defaults to 0.'''}  ,		)
 _lowercase     :					bool      =						field(default=A  ,		metadata={'''help''': '''Benchmark models in eager model.'''} )
 _lowercase     :					bool      =						field(
     default=A  ,		metadata={
         '''help''': '''Benchmark models using XLA JIT compilation. Note that `eager_model` has to be set to `False`.'''
     }  ,		)

 @cached_property
 def    __magic_name__       (		self :	Tuple       ):






      '''simple docstring'''




      requires_backends(self     ,						["tf"]       )
      _lowerCAmelCase   :					Any        =    None
      if self.tpu:
           try:
                if self.tpu_name:
                     _lowerCAmelCase   :					Union[str, Any]        =    tf.distribute.cluster_resolver.TPUClusterResolver(self.tpu_name       )
                else:
                     _lowerCAmelCase   :					List[str]        =    tf.distribute.cluster_resolver.TPUClusterResolver()
           except ValueError:
                _lowerCAmelCase   :					Optional[int]        =    None
      return tpu

 @cached_property
 def    __magic_name__       (		self :	Optional[int]       ):






      '''simple docstring'''




      requires_backends(self     ,						["tf"]       )
      if self.is_tpu:
           tf.config.experimental_connect_to_cluster(self._setup_tpu       )
           tf.tpu.experimental.initialize_tpu_system(self._setup_tpu       )

           _lowerCAmelCase   :					List[Any]        =    tf.distribute.TPUStrategy(self._setup_tpu       )
      else:
           # currently no multi gpu is allowed
           if self.is_gpu:
                # TODO: Currently only single GPU is supported
                tf.config.set_visible_devices(self.gpu_list[self.device_idx]     ,						"GPU"       )
                _lowerCAmelCase   :					Dict        =    tf.distribute.OneDeviceStrategy(device=F'''/gpu:{self.device_idx}'''       )
           else:
                tf.config.set_visible_devices([]     ,						"GPU"       )  # disable GPU
                _lowerCAmelCase   :					Optional[Any]        =    tf.distribute.OneDeviceStrategy(device=F'''/cpu:{self.device_idx}'''       )

      return strategy

 @property
 def    __magic_name__       (		self :	int       ):






      '''simple docstring'''




      requires_backends(self     ,						["tf"]       )
      return self._setup_tpu is not None

 @property
 def    __magic_name__       (		self :	List[Any]       ):






      '''simple docstring'''




      requires_backends(self     ,						["tf"]       )
      return self._setup_strategy

 @property
 def    __magic_name__       (		self :	List[str]       ):






      '''simple docstring'''




      requires_backends(self     ,						["tf"]       )
      return tf.config.list_physical_devices("GPU"       )

 @property
 def    __magic_name__       (		self :	Optional[Any]       ):






      '''simple docstring'''




      requires_backends(self     ,						["tf"]       )
      if self.cuda:
           return len(self.gpu_list       )
      return 0


 @property
 def    __magic_name__       (		self :	Optional[Any]       ):






      '''simple docstring'''




      return self.n_gpu > 0






