






"""simple docstring"""



import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
						import torch


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


A_    =							logging.get_logger(__name__)
A_    =							{'''vocab_file''': '''spiece.model'''}

A_    =							{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

A_    =							{
    '''AI-Sweden/gpt-sw3-126m''': 2048,
    '''AI-Sweden/gpt-sw3-350m''': 2048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2048,
    '''AI-Sweden/gpt-sw3-20b''': 2048,
}






class 							__SCREAMING_SNAKE_CASE						(				UpperCamelCase		):
							snake_case_     =							VOCAB_FILES_NAMES
							snake_case_     =							PRETRAINED_VOCAB_FILES_MAP
							snake_case_     =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case_     =							['input_ids', 'attention_mask']


							def __init__(			self      :						Any			,    snake_case      :						List[Any]			,    snake_case      :						Any=False			,    snake_case      :						int=False			,    snake_case      :						Union[str, Any]=False			,    snake_case      :						Optional[int]=None			,    snake_case      :						Tuple=None			,    snake_case      :						Optional[int]=None			,    snake_case      :						Union[str, Any]=None			,    snake_case      :						Optional[Dict[str, Any]] = None			,    **snake_case      :						str			,    ):
													'''simple docstring'''





													A__						:  List[str]		  =  {} if sp_model_kwargs is None else sp_model_kwargs

													A__						:  Any		  =  kwargs.get("""name_or_path"""							)
													if name_or_path is None:
																			logger.warning(
																			    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
																			    """ you are testing the model, this can safely be ignored"""							)
																			A__						:  List[Any]		  =  """None"""

													# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
													A__						:  List[Any]		  =  """<|endoftext|>""" if eos_token is None else eos_token
													A__						:  str		  =  """<unk>""" if unk_token is None else unk_token
													if "gpt-sw3-7b" in name_or_path:
																			A__						:  int		  =  unk_token if pad_token is None else pad_token
																			A__						:  List[str]		  =  eos_token if bos_token is None else bos_token
													else:
																			A__						:  Optional[Any]		  =  """<pad>""" if pad_token is None else pad_token
																			A__						:  int		  =  """<s>""" if bos_token is None else bos_token

													super().__init__(
													    do_lower_case=snake_case			,    remove_space=snake_case			,    keep_accents=snake_case			,    bos_token=snake_case			,    eos_token=snake_case			,    unk_token=snake_case			,    pad_token=snake_case			,    sp_model_kwargs=self.sp_model_kwargs			,    **snake_case			,    )

													A__						:  List[str]		  =  do_lower_case
													A__						:  int		  =  remove_space
													A__						:  Any		  =  keep_accents
													A__						:  List[Any]		  =  vocab_file

													A__						:  Optional[int]		  =  spm.SentencePieceProcessor(**self.sp_model_kwargs							)
													self.sp_model.Load(snake_case							)

													# Used for whitespace normalization in input texts
													# fmt : off
													A__						:  Optional[int]		  =  {""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
													# fmt : on

													# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
													A__						:  Union[str, Any]		  =  re.compile(
													    F'[{"".join(map(snake_case			,    list(range(0			,    9							)							) + list(range(11			,    32							)							) + list(range(127			,    160							)							) + [160, 173, 8203]							)							)}]'							)


							def __getstate__(			self      :						str							):
													'''simple docstring'''





													A__						:  Dict		  =  self.__dict__.copy()
													A__						:  Tuple		  =  None
													return state


							def __setstate__(			self      :						List[str]			,    snake_case      :						Tuple							):
													'''simple docstring'''





													A__						:  List[str]		  =  d

													# for backward compatibility
													if not hasattr(self			,    """sp_model_kwargs"""							):
																			A__						:  Dict		  =  {}

													A__						:  int		  =  spm.SentencePieceProcessor(**self.sp_model_kwargs							)
													self.sp_model.Load(self.vocab_file							)


							@property
							# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
							def 						_UpperCamelCase       (			self      :						Union[str, Any]							):
													'''simple docstring'''





													return len(self.sp_model							)


							def 						_UpperCamelCase       (			self      :						List[str]			,    snake_case      :						str							):
													'''simple docstring'''





													A__						:  str		  =  self.non_printing_characters_re.sub(""""""			,    snake_case							)

													# Normalize whitespaces
													A__						:  Union[str, Any]		  =  """""".join([char if char not in self.whitespaces else """ """ for char in text]							)

													# NFC Unicode normalization
													A__						:  List[str]		  =  unicodedata.normalize("""NFC"""			,    snake_case							)
													return text


							def 						_UpperCamelCase       (			self      :						List[Any]			,    snake_case      :						str			,    **snake_case      :						int							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  self.preprocess_text(snake_case							)
													return self.sp_model.encode(snake_case			,    out_type=snake_case							)


							def 						_UpperCamelCase       (			self      :						List[Any]			,    snake_case      :						str							):
													'''simple docstring'''





													return self.sp_model.PieceToId(snake_case							)


							def 						_UpperCamelCase       (			self      :						Optional[int]			,    snake_case      :						int							):
													'''simple docstring'''





													return self.sp_model.IdToPiece(snake_case							)


							@staticmethod
							def 						_UpperCamelCase       (			snake_case      :						str							):
													'''simple docstring'''





													return out_string


							def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						List[str]							):
													'''simple docstring'''





													A__						:  List[Any]		  =  []
													A__						:  Tuple		  =  """"""
													A__						:  Union[str, Any]		  =  False
													for token in tokens:
																			# make sure that special tokens are not decoded using sentencepiece model
																			if token in self.all_special_tokens:
																									# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																									if not prev_is_special:
																															out_string += " "

																									out_string += self.sp_model.decode(snake_case							) + token
																									A__						:  List[str]		  =  True
																									A__						:  List[str]		  =  []
																			else:
																									current_sub_tokens.append(snake_case							)
																									A__						:  Any		  =  False
													out_string += self.sp_model.decode(snake_case							)

													return out_string


							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  Any		  =  {self.convert_ids_to_tokens(snake_case							): i for i in range(self.vocab_size							)}
													vocab.update(self.added_tokens_encoder							)
													return vocab


							def 						_UpperCamelCase       (			self      :						Optional[Any]			,    snake_case      :						str			,    snake_case      :						Optional[str] = None							):
													'''simple docstring'''





													if not os.path.isdir(snake_case							):
																			logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
																			return
													A__						:  Optional[Any]		  =  os.path.join(
													    snake_case			,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

													if os.path.abspath(self.vocab_file							) != os.path.abspath(snake_case							) and os.path.isfile(self.vocab_file							):
																			copyfile(self.vocab_file			,    snake_case							)
													elif not os.path.isfile(self.vocab_file							):
																			with open(snake_case			,    """wb"""							) as fi:
																									A__						:  str		  =  self.sp_model.serialized_model_proto()
																									fi.write(snake_case							)

													return (out_vocab_file,)


							def 						_UpperCamelCase       (			self      :						Tuple			,    snake_case      :						Union[str, List[str]]			,    snake_case      :						Union[str, bool] = False							):
													'''simple docstring'''





													if isinstance(snake_case			,    snake_case							):
																			A__						:  str		  =  self.preprocess_text(snake_case							)
																			A__						:  Dict		  =  self.sp_model.encode(snake_case							)
													else:
																			A__						:  Union[str, Any]		  =  [self.preprocess_text(snake_case							) for t in text]
																			A__						:  Any		  =  self.sp_model.encode(snake_case							)

													if return_tensors is True or return_tensors == "pt":
																			A__						:  Optional[int]		  =  torch.tensor(snake_case							)

													return token_ids


							def 						_UpperCamelCase       (			self      :						Optional[Any]			,    snake_case      :						Union[int, List[int]]							):
													'''simple docstring'''





													return self.sp_model.decode(snake_case							)


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						"Conversation"							):
													'''simple docstring'''





													A__						:  Dict		  =  [F'User: {text}' if is_user else F'Bot: {text}' for is_user, text in conversation.iter_texts()]
													A__						:  Optional[int]		  =  (
													    F'{self.eos_token}{self.bos_token}' + F'{self.bos_token}'.join(snake_case							) + F'{self.bos_token}Bot:'
													)
													return self.encode(text=snake_case							)














"""simple docstring"""



import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
A_    =							{'''LayoutLMv2Config''', '''LayoutLMv3Config'''}






@is_pipeline_test
class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):
							snake_case_     =							MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
							snake_case_     =							TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

							if model_mapping is not None:
														snake_case_     =							{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
							if tf_model_mapping is not None:
														snake_case_     =							{
														    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
														}


							@require_torch
							def 						_UpperCamelCase       (			self      :						Tuple							):
													'''simple docstring'''





													A__						:  Tuple		  =  pipeline(
													    task="""text-classification"""			,    model="""hf-internal-testing/tiny-random-distilbert"""			,    framework="""pt"""							)

													A__						:  Dict		  =  text_classifier("""This is great !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}]							)

													A__						:  Optional[int]		  =  text_classifier("""This is great !"""			,    top_k=2							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}]							)

													A__						:  Any		  =  text_classifier(["""This is great !""", """This is bad"""]			,    top_k=2							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [
													        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
													        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
													    ]			,    )

													A__						:  Optional[Any]		  =  text_classifier("""This is great !"""			,    top_k=1							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}]							)

													# Legacy behavior
													A__						:  str		  =  text_classifier("""This is great !"""			,    return_all_scores=snake_case							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}]							)

													A__						:  int		  =  text_classifier("""This is great !"""			,    return_all_scores=snake_case							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [[{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}]]							)

													A__						:  str		  =  text_classifier(["""This is great !""", """Something else"""]			,    return_all_scores=snake_case							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [
													        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
													        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
													    ]			,    )

													A__						:  Optional[int]		  =  text_classifier(["""This is great !""", """Something else"""]			,    return_all_scores=snake_case							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [
													        {"""label""": """LABEL_0""", """score""": 0.504},
													        {"""label""": """LABEL_0""", """score""": 0.504},
													    ]			,    )


							@require_torch
							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													import torch

													A__						:  Tuple		  =  pipeline(
													    task="""text-classification"""			,    model="""hf-internal-testing/tiny-random-distilbert"""			,    framework="""pt"""			,    device=torch.device("""cpu"""							)			,    )

													A__						:  List[str]		  =  text_classifier("""This is great !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}]							)


							@require_tf
							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  str		  =  pipeline(
													    task="""text-classification"""			,    model="""hf-internal-testing/tiny-random-distilbert"""			,    framework="""tf"""							)

													A__						:  Optional[Any]		  =  text_classifier("""This is great !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}]							)


							@slow
							@require_torch
							def 						_UpperCamelCase       (			self      :						List[Any]							):
													'''simple docstring'''





													A__						:  Optional[int]		  =  pipeline("""text-classification"""							)

													A__						:  Optional[int]		  =  text_classifier("""This is great !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """POSITIVE""", """score""": 1.0}]							)
													A__						:  Optional[Any]		  =  text_classifier("""This is bad !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """NEGATIVE""", """score""": 1.0}]							)
													A__						:  Dict		  =  text_classifier("""Birds are a type of animal"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """POSITIVE""", """score""": 0.988}]							)


							@slow
							@require_tf
							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  pipeline("""text-classification"""			,    framework="""tf"""							)

													A__						:  Dict		  =  text_classifier("""This is great !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """POSITIVE""", """score""": 1.0}]							)
													A__						:  Optional[int]		  =  text_classifier("""This is bad !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """NEGATIVE""", """score""": 1.0}]							)
													A__						:  Tuple		  =  text_classifier("""Birds are a type of animal"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """POSITIVE""", """score""": 0.988}]							)


							def 						_UpperCamelCase       (			self      :						Optional[Any]			,    snake_case      :						Any			,    snake_case      :						List[Any]			,    snake_case      :						Any							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  TextClassificationPipeline(model=snake_case			,    tokenizer=snake_case							)
													return text_classifier, ["HuggingFace is in", "This is another test"]


							def 						_UpperCamelCase       (			self      :						int			,    snake_case      :						Union[str, Any]			,    snake_case      :						Dict							):
													'''simple docstring'''





													A__						:  List[str]		  =  text_classifier.model
													# Small inputs because BartTokenizer tiny has maximum position embeddings = 22
													A__						:  int		  =  """HuggingFace is in"""
													A__						:  List[Any]		  =  text_classifier(snake_case							)

													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}]							)
													self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()							)

													A__						:  Any		  =  ["""HuggingFace is in """, """Paris is in France"""]
													A__						:  Any		  =  text_classifier(snake_case							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [{"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}, {"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}]			,    )
													self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()							)
													self.assertTrue(outputs[1]["""label"""] in model.config.idalabel.values()							)

													# Forcing to get all results with `top_k=None`
													# This is NOT the legacy format
													A__						:  List[Any]		  =  text_classifier(snake_case			,    top_k=snake_case							)
													A__						:  Tuple		  =  len(model.config.idalabel.values()							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [[{"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}] * N, [{"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}] * N]			,    )

													A__						:  Optional[Any]		  =  {"""text""": """HuggingFace is in """, """text_pair""": """Paris is in France"""}
													A__						:  Union[str, Any]		  =  text_classifier(snake_case							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    {"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}			,    )
													self.assertTrue(outputs["""label"""] in model.config.idalabel.values()							)

													# This might be used a text pair, but tokenizer + pipe interaction
													# makes it hard to understand that it's not using the pair properly
													# https://github.com/huggingface/transformers/issues/17305
													# We disabled this usage instead as it was outputting wrong outputs.
													A__						:  Tuple		  =  [["""HuggingFace is in """, """Paris is in France"""]]
													with self.assertRaises(snake_case							):
																			text_classifier(snake_case							)

													# This used to be valid for doing text pairs
													# We're keeping it working because of backward compatibility
													A__						:  List[Any]		  =  text_classifier([[["""HuggingFace is in """, """Paris is in France"""]]]							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [{"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}]			,    )
													self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()							)






