





'''simple docstring'''



import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 							__lowercase       (      lowerCAmelCase__ ,      unittest.TestCase						):







	'''simple docstring'''

	a    :				Optional[Any]											=       OpenAIGPTTokenizer
	a    :				int											=       OpenAIGPTTokenizerFast
	a    :				Tuple											=       True
	a    :				Any											=       False


	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				super().setUp()

				# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
				__lowercase										=							[
				    '''l''',
				    '''o''',
				    '''w''',
				    '''e''',
				    '''r''',
				    '''s''',
				    '''t''',
				    '''i''',
				    '''d''',
				    '''n''',
				    '''w</w>''',
				    '''r</w>''',
				    '''t</w>''',
				    '''lo''',
				    '''low''',
				    '''er</w>''',
				    '''low</w>''',
				    '''lowest</w>''',
				    '''newer</w>''',
				    '''wider</w>''',
				    '''<unk>''',
				]
				__lowercase										=							dict(zip(_lowerCamelCase     ,range(len(_lowerCamelCase ) ) ) )
				__lowercase										=							['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

				__lowercase										=							os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES['''vocab_file'''] )
				__lowercase										=							os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES['''merges_file'''] )
				with open(self.vocab_file     ,'''w''' ) as fp:
							fp.write(json.dumps(_lowerCamelCase ) )
				with open(self.merges_file     ,'''w''' ) as fp:
							fp.write('''\n'''.join(_lowerCamelCase ) )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase )							->							Optional[Any]:







				'''simple docstring'''





				return "lower newer", "lower newer"


	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							OpenAIGPTTokenizer(self.vocab_file     ,self.merges_file )

				__lowercase										=							'''lower'''
				__lowercase										=							['''low''', '''er</w>''']
				__lowercase										=							tokenizer.tokenize(_lowerCamelCase )
				self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							tokens + ['''<unk>''']
				__lowercase										=							[14, 15, 20]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase )     ,_lowerCamelCase )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase=15 )							->							str:







				'''simple docstring'''





				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
										__lowercase										=							self.rust_tokenizer_class.from_pretrained(_lowerCamelCase     ,**_lowerCamelCase )

										# Simple input
										__lowercase										=							'''This is a simple input'''
										__lowercase										=							['''This is a simple input 1''', '''This is a simple input 2''']
										__lowercase										=							('''This is a simple input''', '''This is a pair''')
										__lowercase										=							[
										    ('''This is a simple input 1''', '''This is a simple input 2'''),
										    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
										]

										# Simple input tests
										self.assertRaises(_lowerCamelCase     ,tokenizer_r.encode     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='''max_length''' )

										# Simple input
										self.assertRaises(_lowerCamelCase     ,tokenizer_r.encode_plus     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='''max_length''' )

										# Simple input
										self.assertRaises(
										    _lowerCamelCase     ,tokenizer_r.batch_encode_plus     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='''max_length'''     ,)

										# Pair input
										self.assertRaises(_lowerCamelCase     ,tokenizer_r.encode     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='''max_length''' )

										# Pair input
										self.assertRaises(_lowerCamelCase     ,tokenizer_r.encode_plus     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='''max_length''' )

										# Pair input
										self.assertRaises(
										    _lowerCamelCase     ,tokenizer_r.batch_encode_plus     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='''max_length'''     ,)


	def 						_UpperCAmelCase     (self )							->							Optional[int]:







				'''simple docstring'''





				pass






@require_ftfy
@require_spacy
@require_tokenizers
class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	pass









'''simple docstring'''



# flake8: noqa
# Lint as: python3
_SCREAMING_SNAKE_CASE					=    [
    '''VerificationMode''',
    '''Version''',
    '''disable_progress_bar''',
    '''enable_progress_bar''',
    '''is_progress_bar_enabled''',
    '''experimental''',
]

from .info_utils import VerificationMode
from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled
from .version import Version
from .experimental import experimental





