





"""simple docstring"""






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
				import torch


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


lowerCAmelCase_			=							logging.get_logger(__name__)
lowerCAmelCase_			=							{"vocab_file": "spiece.model"}

lowerCAmelCase_			=							{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

lowerCAmelCase_			=							{
    "AI-Sweden/gpt-sw3-126m": 2_048,
    "AI-Sweden/gpt-sw3-350m": 2_048,
    "AI-Sweden/gpt-sw3-1.6b": 2_048,
    "AI-Sweden/gpt-sw3-6.7b": 2_048,
    "AI-Sweden/gpt-sw3-20b": 2_048,
}



class 	lowerCAmelCase					(      UpperCAmelCase_							):
							lowerCAmelCase__						 =  VOCAB_FILES_NAMES
							lowerCAmelCase__						 =  PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase__						 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase__						 =  ["""input_ids""", """attention_mask"""]






							def __init__(   self    ,  a__    ,  a__=False    ,  a__=False    ,  a__=False    ,  a__=None    ,  a__=None    ,  a__=None    ,  a__=None    ,  a__ = None    ,  **a__    ,  ):
														_UpperCAmelCase     =     {} if sp_model_kwargs is None else sp_model_kwargs

														_UpperCAmelCase     =     kwargs.get('name_or_path'     )
														if name_or_path is None:
																					logger.warning(
																					    'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
																					    ' you are testing the model, this can safely be ignored'     )
																					_UpperCAmelCase     =     'None'

														# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
														_UpperCAmelCase     =     '<|endoftext|>' if eos_token is None else eos_token
														_UpperCAmelCase     =     '<unk>' if unk_token is None else unk_token
														if "gpt-sw3-7b" in name_or_path:
																					_UpperCAmelCase     =     unk_token if pad_token is None else pad_token
																					_UpperCAmelCase     =     eos_token if bos_token is None else bos_token
														else:
																					_UpperCAmelCase     =     '<pad>' if pad_token is None else pad_token
																					_UpperCAmelCase     =     '<s>' if bos_token is None else bos_token

														super().__init__(
														    do_lower_case=_lowercase    ,  remove_space=_lowercase    ,  keep_accents=_lowercase    ,  bos_token=_lowercase    ,  eos_token=_lowercase    ,  unk_token=_lowercase    ,  pad_token=_lowercase    ,  sp_model_kwargs=self.sp_model_kwargs    ,  **_lowercase    ,  )

														_UpperCAmelCase     =     do_lower_case
														_UpperCAmelCase     =     remove_space
														_UpperCAmelCase     =     keep_accents
														_UpperCAmelCase     =     vocab_file

														_UpperCAmelCase     =     spm.SentencePieceProcessor(**self.sp_model_kwargs     )
														self.sp_model.Load(_lowercase     )

														# Used for whitespace normalization in input texts
														# fmt : off
														_UpperCAmelCase     =     {' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
														# fmt : on

														# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
														_UpperCAmelCase     =     re.compile(
														    f"""[{"".join(map(_lowercase    ,  list(range(0    ,  9     )     ) + list(range(11    ,  32     )     ) + list(range(1_27    ,  1_60     )     ) + [1_60, 1_73, 82_03]     )     )}]"""     )






							def __getstate__(   self     ):
														_UpperCAmelCase     =     self.__dict__.copy()
														_UpperCAmelCase     =     None
														return state






							def __setstate__(   self    ,  a__     ):
														_UpperCAmelCase     =     d

														# for backward compatibility
														if not hasattr(self    ,  'sp_model_kwargs'     ):
																					_UpperCAmelCase     =     {}

														_UpperCAmelCase     =     spm.SentencePieceProcessor(**self.sp_model_kwargs     )
														self.sp_model.Load(self.vocab_file     )






							@property
							# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
							def 							__A (   self     ):
														return len(self.sp_model     )






							def 							__A (   self    ,  a__     ):
														_UpperCAmelCase     =     self.non_printing_characters_re.sub(''    ,  _lowercase     )

														# Normalize whitespaces
														_UpperCAmelCase     =     ''.join([char if char not in self.whitespaces else ' ' for char in text]     )

														# NFC Unicode normalization
														_UpperCAmelCase     =     unicodedata.normalize('NFC'    ,  _lowercase     )
														return text






							def 							__A (   self    ,  a__    ,  **a__     ):
														_UpperCAmelCase     =     self.preprocess_text(_lowercase     )
														return self.sp_model.encode(_lowercase    ,  out_type=_lowercase     )






							def 							__A (   self    ,  a__     ):
														return self.sp_model.PieceToId(_lowercase     )






							def 							__A (   self    ,  a__     ):
														return self.sp_model.IdToPiece(_lowercase     )






							@staticmethod
							def 							__A (   a__     ):
														return out_string






							def 							__A (   self    ,  a__     ):
														_UpperCAmelCase     =     []
														_UpperCAmelCase     =     ''
														_UpperCAmelCase     =     False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																												if not prev_is_special:
																																			out_string += " "

																												out_string += self.sp_model.decode(_lowercase     ) + token
																												_UpperCAmelCase     =     True
																												_UpperCAmelCase     =     []
																					else:
																												current_sub_tokens.append(_lowercase     )
																												_UpperCAmelCase     =     False
														out_string += self.sp_model.decode(_lowercase     )

														return out_string






							def 							__A (   self     ):
														_UpperCAmelCase     =     {self.convert_ids_to_tokens(_lowercase     ): i for i in range(self.vocab_size     )}
														vocab.update(self.added_tokens_encoder     )
														return vocab






							def 							__A (   self    ,  a__    ,  a__ = None     ):
														if not os.path.isdir(_lowercase     ):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
																					return
														_UpperCAmelCase     =     os.path.join(
														    _lowercase    ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )

														if os.path.abspath(self.vocab_file     ) != os.path.abspath(_lowercase     ) and os.path.isfile(self.vocab_file     ):
																					copyfile(self.vocab_file    ,  _lowercase     )
														elif not os.path.isfile(self.vocab_file     ):
																					with open(_lowercase    ,  'wb'     ) as fi:
																												_UpperCAmelCase     =     self.sp_model.serialized_model_proto()
																												fi.write(_lowercase     )

														return (out_vocab_file,)






							def 							__A (   self    ,  a__    ,  a__ = False     ):

														if isinstance(_lowercase    ,  _lowercase     ):
																					_UpperCAmelCase     =     self.preprocess_text(_lowercase     )
																					_UpperCAmelCase     =     self.sp_model.encode(_lowercase     )
														else:
																					_UpperCAmelCase     =     [self.preprocess_text(_lowercase     ) for t in text]
																					_UpperCAmelCase     =     self.sp_model.encode(_lowercase     )

														if return_tensors is True or return_tensors == "pt":
																					_UpperCAmelCase     =     torch.tensor(_lowercase     )

														return token_ids






							def 							__A (   self    ,  a__     ):
														return self.sp_model.decode(_lowercase     )






							def 							__A (   self    ,  a__     ):
														_UpperCAmelCase     =     [f"""User: {text}""" if is_user else f"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
														_UpperCAmelCase     =     (
														    f"""{self.eos_token}{self.bos_token}""" + f"""{self.bos_token}""".join(_lowercase     ) + f"""{self.bos_token}Bot:"""
														)
														return self.encode(text=_lowercase     )













"""simple docstring"""






# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCAmelCase_			=							{'''configuration_timm_backbone''': ['''TimmBackboneConfig''']}


try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCAmelCase_			=							['''TimmBackbone''']


if TYPE_CHECKING:
    from .configuration_timm_backbone import TimmBackboneConfig

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_timm_backbone import TimmBackbone

else:
    import sys

    lowerCAmelCase_			=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






