
import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__							:							Any									=			get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model")






@require_sentencepiece
@require_tokenizers
class        SCREAMING_SNAKE_CASE__    (							SCREAMING_SNAKE_CASE_   ,		unittest.TestCase    ):

							"""simple docstring"""






							_snake_case     		=  GPTSwaTokenizer
							_snake_case     		=  False
							_snake_case     		=  True
							_snake_case     		=  False



							def     A__		(  self			)->   Tuple:





													'''simple docstring'''






													super().setUp()

													# We have a SentencePiece fixture for testing
													__UpperCamelCase       =			GPTSwaTokenizer(SCREAMING_SNAKE_CASE_     ,	eos_token='''<unk>'''     ,	bos_token='''<unk>'''     ,	pad_token='''<unk>'''			)

													tokenizer.save_pretrained(self.tmpdirname			)



							def     A__		(  self     ,	SCREAMING_SNAKE_CASE_			)->   Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase       =			'''This is a test'''
													__UpperCamelCase       =			'''This is a test'''
													return input_text, output_text



							def     A__		(  self			)->   Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase       =			'''<s>'''
													__UpperCamelCase       =			1

													self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE_			)     ,	SCREAMING_SNAKE_CASE_			)
													self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE_			)     ,	SCREAMING_SNAKE_CASE_			)



							def     A__		(  self			)->   str:





													'''simple docstring'''






													__UpperCamelCase       =			list(self.get_tokenizer().get_vocab().keys()			)

													self.assertEqual(vocab_keys[0]     ,	'''<unk>'''			)
													self.assertEqual(vocab_keys[1]     ,	'''<s>'''			)
													self.assertEqual(vocab_keys[-1]     ,	'''j'''			)
													self.assertEqual(len(SCREAMING_SNAKE_CASE_			)     ,	2000			)



							def     A__		(  self			)->   Optional[Any]:





													'''simple docstring'''






													self.assertEqual(self.get_tokenizer().vocab_size     ,	2000			)



							def     A__		(  self			)->   int:





													'''simple docstring'''






													__UpperCamelCase       =			GPTSwaTokenizer(SCREAMING_SNAKE_CASE_			)

													__UpperCamelCase       =			tokenizer.tokenize('''This is a test'''			)
													self.assertListEqual(SCREAMING_SNAKE_CASE_     ,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']			)

													self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_			)     ,	[465, 287, 265, 631, 842]			)

													__UpperCamelCase       =			tokenizer.tokenize('''I was born in 92000, and this is falsé.'''			)
													# fmt: off
													self.assertListEqual(
													    SCREAMING_SNAKE_CASE_     ,	['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']     ,	)
													# fmt: on

													__UpperCamelCase       =			tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_			)
													self.assertListEqual(
													    SCREAMING_SNAKE_CASE_     ,	[262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]     ,	)

													__UpperCamelCase       =			tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_			)
													# fmt: off
													self.assertListEqual(
													    SCREAMING_SNAKE_CASE_     ,	['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']			)
													# fmt: on



							def     A__		(  self			)->   List[Any]:





													'''simple docstring'''






													__UpperCamelCase       =			GPTSwaTokenizer(SCREAMING_SNAKE_CASE_			)
													__UpperCamelCase       =			['''This is a test''', '''I was born in 92000, and this is falsé.''']
													__UpperCamelCase       =			[
													    [465, 287, 265, 631, 842],
													    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
													]

													# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
													for text, expected_ids in zip(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			):
																			self.assertListEqual(tokenizer.encode_fast(SCREAMING_SNAKE_CASE_			)     ,	SCREAMING_SNAKE_CASE_			)

													# Test that decode_fast returns the input text
													for text, token_ids in zip(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			):
																			self.assertEqual(tokenizer.decode_fast(SCREAMING_SNAKE_CASE_			)     ,	SCREAMING_SNAKE_CASE_			)



							@slow
							def     A__		(  self			)->   Union[str, Any]:





													'''simple docstring'''






													__UpperCamelCase       =			[
													    '''<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')''',
													    '''Hey there, how are you doing this fine day?''',
													    '''This is a text with a trailing spaces followed by a dot     .''',
													    '''Häj sväjs lillebrör! =)''',
													    '''Det är inget fel på Mr. Cool''',
													]

													# fmt: off
													__UpperCamelCase       =			{'''input_ids''': [[63423, 5, 6811, 14954, 282, 816, 3821, 63466, 63425, 63462, 18, 63978, 678, 301, 1320, 63423, 63455, 63458, 18, 63982, 4246, 3940, 1901, 47789, 5547, 18994], [19630, 1100, 63446, 1342, 633, 544, 4488, 593, 5102, 2416, 63495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 58593, 22413, 9106, 546, 268, 33213, 63979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55130, 63450, 924, 63449, 2249, 4062, 1558, 318, 63504, 21498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 63443, 26801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
													# fmt: on
													self.tokenizer_integration_test_util(
													    expected_encoding=SCREAMING_SNAKE_CASE_     ,	model_name='''AI-Sweden/gpt-sw3-126m'''     ,	sequences=SCREAMING_SNAKE_CASE_     ,	)






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


lowercase__							:							Dict									=			{"configuration_vit": ["VIT_PRETRAINED_CONFIG_ARCHIVE_MAP", "ViTConfig", "ViTOnnxConfig"]}

try:
				if not is_vision_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase__							:							Tuple									=			["ViTFeatureExtractor"]
				lowercase__							:							str									=			["ViTImageProcessor"]

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase__							:							List[str]									=			[
				    "VIT_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "ViTForImageClassification",
				    "ViTForMaskedImageModeling",
				    "ViTModel",
				    "ViTPreTrainedModel",
				]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase__							:							Tuple									=			[
				    "TFViTForImageClassification",
				    "TFViTModel",
				    "TFViTPreTrainedModel",
				]

try:
				if not is_flax_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase__							:							List[Any]									=			[
				    "FlaxViTForImageClassification",
				    "FlaxViTModel",
				    "FlaxViTPreTrainedModel",
				]

if TYPE_CHECKING:
				from .configuration_vit import VIT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTConfig, ViTOnnxConfig

				try:
								if not is_vision_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .feature_extraction_vit import ViTFeatureExtractor
								from .image_processing_vit import ViTImageProcessor

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_vit import (
								    VIT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    ViTForImageClassification,
								    ViTForMaskedImageModeling,
								    ViTModel,
								    ViTPreTrainedModel,
								)

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_vit import TFViTForImageClassification, TFViTModel, TFViTPreTrainedModel

				try:
								if not is_flax_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel, FlaxViTPreTrainedModel


else:
				import sys

				lowercase__							:							List[Any]									=			_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




