




import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 							__UpperCamelCase			( __UpperCAmelCase     ,					unittest.TestCase  ):





				'''simple docstring'''



				__a							: Optional[int]										=OpenAIGPTTokenizer
				__a							: Tuple										=OpenAIGPTTokenizerFast
				__a							: Any										=True
				__a							: Union[str, Any]										=False



				def       __snake_case		( self						):
											super().setUp()

											# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
											lowerCAmelCase   =		[
											    '''l''',
											    '''o''',
											    '''w''',
											    '''e''',
											    '''r''',
											    '''s''',
											    '''t''',
											    '''i''',
											    '''d''',
											    '''n''',
											    '''w</w>''',
											    '''r</w>''',
											    '''t</w>''',
											    '''lo''',
											    '''low''',
											    '''er</w>''',
											    '''low</w>''',
											    '''lowest</w>''',
											    '''newer</w>''',
											    '''wider</w>''',
											    '''<unk>''',
											]
											lowerCAmelCase   =		dict(zip(UpperCAmelCase_       , range(len(UpperCAmelCase_						)						)						)						)
											lowerCAmelCase   =		['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

											lowerCAmelCase   =		os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES['''vocab_file''']						)
											lowerCAmelCase   =		os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES['''merges_file''']						)
											with open(self.vocab_file       , '''w'''						) as fp:
																		fp.write(json.dumps(UpperCAmelCase_						)						)
											with open(self.merges_file       , '''w'''						) as fp:
																		fp.write('''\n'''.join(UpperCAmelCase_						)						)



				def       __snake_case		( self       , UpperCAmelCase_						):
											return "lower newer", "lower newer"



				def       __snake_case		( self						):
											lowerCAmelCase   =		OpenAIGPTTokenizer(self.vocab_file       , self.merges_file						)

											lowerCAmelCase   =		'''lower'''
											lowerCAmelCase   =		['''low''', '''er</w>''']
											lowerCAmelCase   =		tokenizer.tokenize(UpperCAmelCase_						)
											self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_						)

											lowerCAmelCase   =		tokens + ['''<unk>''']
											lowerCAmelCase   =		[14, 15, 20]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_						)       , UpperCAmelCase_						)



				def       __snake_case		( self       , UpperCAmelCase_=15						):
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
																									lowerCAmelCase   =		self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_       , **UpperCAmelCase_						)

																									# Simple input
																									lowerCAmelCase   =		'''This is a simple input'''
																									lowerCAmelCase   =		['''This is a simple input 1''', '''This is a simple input 2''']
																									lowerCAmelCase   =		('''This is a simple input''', '''This is a pair''')
																									lowerCAmelCase   =		[
																									    ('''This is a simple input 1''', '''This is a simple input 2'''),
																									    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																									]

																									# Simple input tests
																									self.assertRaises(UpperCAmelCase_       , tokenizer_r.encode       , UpperCAmelCase_       , max_length=UpperCAmelCase_       , padding='''max_length'''						)

																									# Simple input
																									self.assertRaises(UpperCAmelCase_       , tokenizer_r.encode_plus       , UpperCAmelCase_       , max_length=UpperCAmelCase_       , padding='''max_length'''						)

																									# Simple input
																									self.assertRaises(
																									    UpperCAmelCase_       , tokenizer_r.batch_encode_plus       , UpperCAmelCase_       , max_length=UpperCAmelCase_       , padding='''max_length'''       , )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_       , tokenizer_r.encode       , UpperCAmelCase_       , max_length=UpperCAmelCase_       , padding='''max_length'''						)

																									# Pair input
																									self.assertRaises(UpperCAmelCase_       , tokenizer_r.encode_plus       , UpperCAmelCase_       , max_length=UpperCAmelCase_       , padding='''max_length'''						)

																									# Pair input
																									self.assertRaises(
																									    UpperCAmelCase_       , tokenizer_r.batch_encode_plus       , UpperCAmelCase_       , max_length=UpperCAmelCase_       , padding='''max_length'''       , )






				def       __snake_case		( self						):
											pass




@require_ftfy
@require_spacy
@require_tokenizers
class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				pass









import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


UpperCAmelCase_						="""hf-internal-testing/tiny-random-bert"""
UpperCAmelCase_						=os.path.join(TRANSFORMERS_CACHE, """models--hf-internal-testing--tiny-random-bert""")
UpperCAmelCase_						="""9b8c223d42b2188cb49d29af482996f9d0f3e5a6"""


class 							__UpperCamelCase			( unittest.TestCase  ):





				'''simple docstring'''



				def       __snake_case		( self						):
											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_						)
											# Should have downloaded the file in here
											self.assertTrue(os.path.isdir(UpperCAmelCase_						)						)
											# Cache should contain at least those three subfolders:
											for subfolder in ["blobs", "refs", "snapshots"]:
																		self.assertTrue(os.path.isdir(os.path.join(UpperCAmelCase_       , UpperCAmelCase_						)						)						)
											with open(os.path.join(UpperCAmelCase_       , '''refs'''       , '''main'''						)						) as f:
																		lowerCAmelCase   =		f.read()
											self.assertEqual(UpperCAmelCase_       , os.path.join(UpperCAmelCase_       , '''snapshots'''       , UpperCAmelCase_       , UpperCAmelCase_						)						)
											self.assertTrue(os.path.isfile(UpperCAmelCase_						)						)

											# File is cached at the same place the second time.
											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_						)
											self.assertEqual(UpperCAmelCase_       , UpperCAmelCase_						)

											# Using a specific revision to test the full commit hash.
											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_       , revision='''9b8c223'''						)
											self.assertEqual(UpperCAmelCase_       , os.path.join(UpperCAmelCase_       , '''snapshots'''       , UpperCAmelCase_       , UpperCAmelCase_						)						)



				def       __snake_case		( self						):
											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid model identifier'''						):
																		lowerCAmelCase   =		cached_file('''tiny-random-bert'''       , UpperCAmelCase_						)

											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid git identifier'''						):
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_       , revision='''aaaa'''						)

											with self.assertRaisesRegex(UpperCAmelCase_       , '''does not appear to have a file named'''						):
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''						)



				def       __snake_case		( self						):
											with self.assertRaisesRegex(UpperCAmelCase_       , '''does not appear to have a file named'''						):
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''						)

											with open(os.path.join(UpperCAmelCase_       , '''refs'''       , '''main'''						)						) as f:
																		lowerCAmelCase   =		f.read()
											self.assertTrue(os.path.isfile(os.path.join(UpperCAmelCase_       , '''.no_exist'''       , UpperCAmelCase_       , '''conf'''						)						)						)

											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''       , _raise_exceptions_for_missing_entries=UpperCAmelCase_						)
											self.assertIsNone(UpperCAmelCase_						)

											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''       , local_files_only=UpperCAmelCase_       , _raise_exceptions_for_missing_entries=UpperCAmelCase_						)
											self.assertIsNone(UpperCAmelCase_						)

											lowerCAmelCase   =		mock.Mock()
											lowerCAmelCase   =		5_00
											lowerCAmelCase   =		{}
											lowerCAmelCase   =		HTTPError
											lowerCAmelCase   =		{}

											# Under the mock environment we get a 500 error when trying to reach the tokenizer.
											with mock.patch('''requests.Session.request'''       , return_value=UpperCAmelCase_						) as mock_head:
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''       , _raise_exceptions_for_connection_errors=UpperCAmelCase_						)
																		self.assertIsNone(UpperCAmelCase_						)
																		# This check we did call the fake head request
																		mock_head.assert_called()



				def       __snake_case		( self						):
											self.assertTrue(has_file('''hf-internal-testing/tiny-bert-pt-only'''       , UpperCAmelCase_						)						)
											self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only'''       , UpperCAmelCase_						)						)
											self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only'''       , UpperCAmelCase_						)						)



				def       __snake_case		( self						):
											# `get_file_from_repo` returns None if the file does not exist
											self.assertIsNone(get_file_from_repo('''bert-base-cased'''       , '''ahah.txt'''						)						)

											# The function raises if the repository does not exist.
											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid model identifier'''						):
																		get_file_from_repo('''bert-base-case'''       , UpperCAmelCase_						)

											# The function raises if the revision does not exist.
											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid git identifier'''						):
																		get_file_from_repo('''bert-base-cased'''       , UpperCAmelCase_       , revision='''ahaha'''						)

											lowerCAmelCase   =		get_file_from_repo('''bert-base-cased'''       , UpperCAmelCase_						)
											# The name is the cached name which is not very easy to test, so instead we load the content.
											lowerCAmelCase   =		json.loads(open(UpperCAmelCase_       , '''r'''						).read()						)
											self.assertEqual(config['''hidden_size''']       , 7_68						)






				def       __snake_case		( self						):
											with tempfile.TemporaryDirectory() as tmp_dir:
																		lowerCAmelCase   =		Path(UpperCAmelCase_						) / '''a.txt'''
																		filename.touch()
																		self.assertEqual(get_file_from_repo(UpperCAmelCase_       , '''a.txt'''						)       , str(UpperCAmelCase_						)						)

																		self.assertIsNone(get_file_from_repo(UpperCAmelCase_       , '''b.txt'''						)						)



