



'''simple docstring'''





import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    CLIPConfig,
    CLIPImageProcessor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_image_processing import CustomImageProcessor  # noqa E402

class 					_A (			unittest.TestCase					):



       def 	_lowerCamelCase    (	self							:  List[Any]):


           '''simple docstring'''

           __a		 =       0



       def 	_lowerCamelCase    (	self							:  List[str]):


           '''simple docstring'''

           __a		 =       AutoImageProcessor.from_pretrained('''openai/clip-vit-base-patch32''')
           self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE)



       def 	_lowerCamelCase    (	self							:  Dict):


           '''simple docstring'''

           with tempfile.TemporaryDirectory() as tmpdirname:
               __a		 =       Path(__SCREAMING_SNAKE_CASE) / '''preprocessor_config.json'''
               __a		 =       Path(__SCREAMING_SNAKE_CASE) / '''config.json'''
               json.dump(
                   {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}  ,	open(__SCREAMING_SNAKE_CASE  ,	'''w''')  ,	)
               json.dump({'''model_type''': '''clip'''}  ,	open(__SCREAMING_SNAKE_CASE  ,	'''w'''))

               __a		 =       AutoImageProcessor.from_pretrained(__SCREAMING_SNAKE_CASE)
               self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE)



       def 	_lowerCamelCase    (	self							:  Optional[Any]):


           '''simple docstring'''

           with tempfile.TemporaryDirectory() as tmpdirname:
               __a		 =       Path(__SCREAMING_SNAKE_CASE) / '''preprocessor_config.json'''
               __a		 =       Path(__SCREAMING_SNAKE_CASE) / '''config.json'''
               json.dump(
                   {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}  ,	open(__SCREAMING_SNAKE_CASE  ,	'''w''')  ,	)
               json.dump({'''model_type''': '''clip'''}  ,	open(__SCREAMING_SNAKE_CASE  ,	'''w'''))

               __a		 =       AutoImageProcessor.from_pretrained(__SCREAMING_SNAKE_CASE)
               self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE)



       def 	_lowerCamelCase    (	self							:  List[Any]):


           '''simple docstring'''

           with tempfile.TemporaryDirectory() as tmpdirname:
               __a		 =       CLIPConfig()

               # Create a dummy config file with image_proceesor_type
               __a		 =       Path(__SCREAMING_SNAKE_CASE) / '''preprocessor_config.json'''
               __a		 =       Path(__SCREAMING_SNAKE_CASE) / '''config.json'''
               json.dump(
                   {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}  ,	open(__SCREAMING_SNAKE_CASE  ,	'''w''')  ,	)
               json.dump({'''model_type''': '''clip'''}  ,	open(__SCREAMING_SNAKE_CASE  ,	'''w'''))

               # remove image_processor_type to make sure config.json alone is enough to load image processor locally
               __a		 =       AutoImageProcessor.from_pretrained(__SCREAMING_SNAKE_CASE).to_dict()

               config_dict.pop('''image_processor_type''')
               __a		 =       CLIPImageProcessor(**__SCREAMING_SNAKE_CASE)

               # save in new folder
               model_config.save_pretrained(__SCREAMING_SNAKE_CASE)
               config.save_pretrained(__SCREAMING_SNAKE_CASE)

               __a		 =       AutoImageProcessor.from_pretrained(__SCREAMING_SNAKE_CASE)

               # make sure private variable is not incorrectly saved
               __a		 =       json.loads(config.to_json_string())
               self.assertTrue('''_processor_class''' not in dict_as_saved)

           self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE)



       def 	_lowerCamelCase    (	self							:  Optional[Any]):


           '''simple docstring'''

           with tempfile.TemporaryDirectory() as tmpdirname:
               __a		 =       Path(__SCREAMING_SNAKE_CASE) / '''preprocessor_config.json'''
               json.dump(
                   {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}  ,	open(__SCREAMING_SNAKE_CASE  ,	'''w''')  ,	)

               __a		 =       AutoImageProcessor.from_pretrained(__SCREAMING_SNAKE_CASE)
               self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE)



       def 	_lowerCamelCase    (	self							:  Any):


           '''simple docstring'''

           with self.assertRaisesRegex(
               __SCREAMING_SNAKE_CASE  ,	'''clip-base is not a local folder and is not a valid model identifier'''):
               __a		 =       AutoImageProcessor.from_pretrained('''clip-base''')



       def 	_lowerCamelCase    (	self							:  int):


           '''simple docstring'''

           with self.assertRaisesRegex(
               __SCREAMING_SNAKE_CASE  ,	r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''):
               __a		 =       AutoImageProcessor.from_pretrained(__SCREAMING_SNAKE_CASE  ,	revision='''aaaaaa''')



       def 	_lowerCamelCase    (	self							:  int):


           '''simple docstring'''

           with self.assertRaisesRegex(
               __SCREAMING_SNAKE_CASE  ,	'''hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'''  ,	):
               __a		 =       AutoImageProcessor.from_pretrained('''hf-internal-testing/config-no-model''')



       def 	_lowerCamelCase    (	self							:  Tuple):


           '''simple docstring'''

           with self.assertRaises(__SCREAMING_SNAKE_CASE):
               __a		 =       AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor''')
           # If remote code is disabled, we can't load this config.
           with self.assertRaises(__SCREAMING_SNAKE_CASE):
               __a		 =       AutoImageProcessor.from_pretrained(
                   '''hf-internal-testing/test_dynamic_image_processor'''  ,	trust_remote_code=__SCREAMING_SNAKE_CASE)

           __a		 =       AutoImageProcessor.from_pretrained(
               '''hf-internal-testing/test_dynamic_image_processor'''  ,	trust_remote_code=__SCREAMING_SNAKE_CASE)
           self.assertEqual(image_processor.__class__.__name__  ,	'''NewImageProcessor''')

           # Test image processor can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
               image_processor.save_pretrained(__SCREAMING_SNAKE_CASE)
               __a		 =       AutoImageProcessor.from_pretrained(__SCREAMING_SNAKE_CASE  ,	trust_remote_code=__SCREAMING_SNAKE_CASE)
           self.assertEqual(reloaded_image_processor.__class__.__name__  ,	'''NewImageProcessor''')



       def 	_lowerCamelCase    (	self							:  Any):


           '''simple docstring'''

           try:
               AutoConfig.register('''custom'''  ,	__SCREAMING_SNAKE_CASE)
               AutoImageProcessor.register(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE)
               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(__SCREAMING_SNAKE_CASE):
                   AutoImageProcessor.register(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE)

               with tempfile.TemporaryDirectory() as tmpdirname:
                   __a		 =       Path(__SCREAMING_SNAKE_CASE) / '''preprocessor_config.json'''
                   __a		 =       Path(__SCREAMING_SNAKE_CASE) / '''config.json'''
                   json.dump(
                       {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}  ,	open(__SCREAMING_SNAKE_CASE  ,	'''w''')  ,	)
                   json.dump({'''model_type''': '''clip'''}  ,	open(__SCREAMING_SNAKE_CASE  ,	'''w'''))

                   __a		 =       CustomImageProcessor.from_pretrained(__SCREAMING_SNAKE_CASE)

               # Now that the config is registered, it can be used as any other config with the auto-API
               with tempfile.TemporaryDirectory() as tmp_dir:
                   image_processor.save_pretrained(__SCREAMING_SNAKE_CASE)
                   __a		 =       AutoImageProcessor.from_pretrained(__SCREAMING_SNAKE_CASE)
                   self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE)

           finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
                   del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]





       def 	_lowerCamelCase    (	self							:  Optional[int]):


           '''simple docstring'''

           class 					_A (			__UpperCAmelCase					):
                  UpperCamelCase__						:					Dict		      =		True

           try:
               AutoConfig.register('''custom'''  ,	__SCREAMING_SNAKE_CASE)
               AutoImageProcessor.register(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE)
               # If remote code is not set, the default is to use local
               __a		 =       AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor''')
               self.assertEqual(image_processor.__class__.__name__  ,	'''NewImageProcessor''')
               self.assertTrue(image_processor.is_local)

               # If remote code is disabled, we load the local one.
               __a		 =       AutoImageProcessor.from_pretrained(
                   '''hf-internal-testing/test_dynamic_image_processor'''  ,	trust_remote_code=__SCREAMING_SNAKE_CASE)
               self.assertEqual(image_processor.__class__.__name__  ,	'''NewImageProcessor''')
               self.assertTrue(image_processor.is_local)

               # If remote is enabled, we load from the Hub
               __a		 =       AutoImageProcessor.from_pretrained(
                   '''hf-internal-testing/test_dynamic_image_processor'''  ,	trust_remote_code=__SCREAMING_SNAKE_CASE)
               self.assertEqual(image_processor.__class__.__name__  ,	'''NewImageProcessor''')
               self.assertTrue(not hasattr(__SCREAMING_SNAKE_CASE  ,	'''is_local'''))

           finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
                   del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]










import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case		:Dict   	=  get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')

@require_sentencepiece
@require_tokenizers
class 					_A (			__UpperCAmelCase   ,unittest.TestCase					):
       UpperCamelCase__						:					List[str]		      =		GPTSwaTokenizer
       UpperCamelCase__						:					Dict		      =		False
       UpperCamelCase__						:					int		      =		True
       UpperCamelCase__						:					List[Any]		      =		False



       def 	_lowerCamelCase    (	self							:  List[Any]):


           '''simple docstring'''

           super().setUp()

           # We have a SentencePiece fixture for testing
           __a		 =       GPTSwaTokenizer(__SCREAMING_SNAKE_CASE  ,	eos_token='''<unk>'''  ,	bos_token='''<unk>'''  ,	pad_token='''<unk>''')

           tokenizer.save_pretrained(self.tmpdirname)



       def 	_lowerCamelCase    (	self							:  Tuple  ,	__SCREAMING_SNAKE_CASE							:  int):


           '''simple docstring'''

           __a		 =       '''This is a test'''
           __a		 =       '''This is a test'''
           return input_text, output_text



       def 	_lowerCamelCase    (	self							:  Dict):


           '''simple docstring'''

           __a		 =       '''<s>'''
           __a		 =       1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(__SCREAMING_SNAKE_CASE)  ,	__SCREAMING_SNAKE_CASE)
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(__SCREAMING_SNAKE_CASE)  ,	__SCREAMING_SNAKE_CASE)



       def 	_lowerCamelCase    (	self							:  List[Any]):


           '''simple docstring'''

           __a		 =       list(self.get_tokenizer().get_vocab().keys())

           self.assertEqual(vocab_keys[0]  ,	'''<unk>''')
           self.assertEqual(vocab_keys[1]  ,	'''<s>''')
           self.assertEqual(vocab_keys[-1]  ,	'''j''')
           self.assertEqual(len(__SCREAMING_SNAKE_CASE)  ,	2_000)



       def 	_lowerCamelCase    (	self							:  Dict):


           '''simple docstring'''

           self.assertEqual(self.get_tokenizer().vocab_size  ,	2_000)



       def 	_lowerCamelCase    (	self							:  List[str]):


           '''simple docstring'''

           __a		 =       GPTSwaTokenizer(__SCREAMING_SNAKE_CASE)

           __a		 =       tokenizer.tokenize('''This is a test''')
           self.assertListEqual(__SCREAMING_SNAKE_CASE  ,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

           self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE)  ,	[465, 287, 265, 631, 842])

           __a		 =       tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
           # fmt: off
           self.assertListEqual(
               __SCREAMING_SNAKE_CASE  ,	['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']  ,	)
           # fmt: on

           __a		 =       tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE)
           self.assertListEqual(
               __SCREAMING_SNAKE_CASE  ,	[262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]  ,	)

           __a		 =       tokenizer.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE)
           # fmt: off
           self.assertListEqual(
               __SCREAMING_SNAKE_CASE  ,	['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.'''])
           # fmt: on



       def 	_lowerCamelCase    (	self							:  Any):


           '''simple docstring'''

           __a		 =       GPTSwaTokenizer(__SCREAMING_SNAKE_CASE)
           __a		 =       ['''This is a test''', '''I was born in 92000, and this is falsé.''']
           __a		 =       [
               [465, 287, 265, 631, 842],
               [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
           ]

           # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
           for text, expected_ids in zip(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE):
               self.assertListEqual(tokenizer.encode_fast(__SCREAMING_SNAKE_CASE)  ,	__SCREAMING_SNAKE_CASE)

           # Test that decode_fast returns the input text
           for text, token_ids in zip(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE):
               self.assertEqual(tokenizer.decode_fast(__SCREAMING_SNAKE_CASE)  ,	__SCREAMING_SNAKE_CASE)





       @slow
       def 	_lowerCamelCase    (	self							:  Any):


           '''simple docstring'''

           __a		 =       [
               '''<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')''',
               '''Hey there, how are you doing this fine day?''',
               '''This is a text with a trailing spaces followed by a dot     .''',
               '''Häj sväjs lillebrör! =)''',
               '''Det är inget fel på Mr. Cool''',
           ]

           # fmt: off
           __a		 =       {'''input_ids''': [[63_423, 5, 6_811, 14_954, 282, 816, 3_821, 63_466, 63_425, 63_462, 18, 63_978, 678, 301, 1_320, 63_423, 63_455, 63_458, 18, 63_982, 4_246, 3_940, 1_901, 47_789, 5_547, 18_994], [19_630, 1_100, 63_446, 1_342, 633, 544, 4_488, 593, 5_102, 2_416, 63_495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_652, 428, 268, 1_936, 515, 268, 58_593, 22_413, 9_106, 546, 268, 33_213, 63_979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55_130, 63_450, 924, 63_449, 2_249, 4_062, 1_558, 318, 63_504, 21_498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2_827, 2_559, 332, 6_575, 63_443, 26_801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
           # fmt: on
           self.tokenizer_integration_test_util(
               expected_encoding=__SCREAMING_SNAKE_CASE  ,	model_name='''AI-Sweden/gpt-sw3-126m'''  ,	sequences=__SCREAMING_SNAKE_CASE  ,	)





