import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class       lowercase__		(     UpperCamelCase_			,							unittest.TestCase):
						UpperCamelCase_  =	GPTaTokenizer
						UpperCamelCase_  =	GPTaTokenizerFast
						UpperCamelCase_  =	True
						UpperCamelCase_  =	{"""add_prefix_space""": True}
						UpperCamelCase_  =	False


						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												SCREAMING_SNAKE_CASE							:					List[str]           =							[
												    '''l''',
												    '''o''',
												    '''w''',
												    '''e''',
												    '''r''',
												    '''s''',
												    '''t''',
												    '''i''',
												    '''d''',
												    '''n''',
												    '''\u0120''',
												    '''\u0120l''',
												    '''\u0120n''',
												    '''\u0120lo''',
												    '''\u0120low''',
												    '''er''',
												    '''\u0120lowest''',
												    '''\u0120newer''',
												    '''\u0120wider''',
												    '''<unk>''',
												    '''<|endoftext|>''',
												]
												SCREAMING_SNAKE_CASE							:					Tuple           =							dict(zip(UpperCamelCase__					,	range(len(UpperCamelCase__		)		)		)		)
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
												SCREAMING_SNAKE_CASE							:					int           =							{'''unk_token''': '''<unk>'''}

												SCREAMING_SNAKE_CASE							:					int           =							os.path.join(self.tmpdirname					,	VOCAB_FILES_NAMES['''vocab_file''']		)
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							os.path.join(self.tmpdirname					,	VOCAB_FILES_NAMES['''merges_file''']		)
												with open(self.vocab_file					,	'''w'''					,	encoding='''utf-8'''		) as fp:
																		fp.write(json.dumps(UpperCamelCase__		) + '''\n'''		)
												with open(self.merges_file					,	'''w'''					,	encoding='''utf-8'''		) as fp:
																		fp.write('''\n'''.join(UpperCamelCase__		)		)


						def  __A ( self			:       Any					,	**UpperCamelCase__			:       Dict		):





												'''simple docstring'''





												kwargs.update(self.special_tokens_map		)
												return GPTaTokenizer.from_pretrained(self.tmpdirname					,	**UpperCamelCase__		)


						def  __A ( self			:       Optional[Any]					,	**UpperCamelCase__			:       Any		):





												'''simple docstring'''





												kwargs.update(self.special_tokens_map		)
												return GPTaTokenizerFast.from_pretrained(self.tmpdirname					,	**UpperCamelCase__		)


						def  __A ( self			:       str					,	UpperCamelCase__			:       Dict		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					str           =							'''lower newer'''
												SCREAMING_SNAKE_CASE							:					List[str]           =							'''lower newer'''
												return input_text, output_text


						def  __A ( self			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					List[str]           =							GPTaTokenizer(self.vocab_file					,	self.merges_file					,	**self.special_tokens_map		)
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							'''lower newer'''
												SCREAMING_SNAKE_CASE							:					str           =							['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokenizer.tokenize(UpperCamelCase__					,	add_prefix_space=UpperCamelCase__		)
												self.assertListEqual(UpperCamelCase__					,	UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					int           =							tokens + [tokenizer.unk_token]
												SCREAMING_SNAKE_CASE							:					List[str]           =							[14, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__		)					,	UpperCamelCase__		)


						def  __A ( self			:       str		):





												'''simple docstring'''





												if not self.test_rust_tokenizer:
																		return

												SCREAMING_SNAKE_CASE							:					List[Any]           =							self.get_tokenizer()
												SCREAMING_SNAKE_CASE							:					List[Any]           =							self.get_rust_tokenizer(add_prefix_space=UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					Dict           =							'''lower newer'''

												# Testing tokenization
												SCREAMING_SNAKE_CASE							:					str           =							tokenizer.tokenize(UpperCamelCase__					,	add_prefix_space=UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							rust_tokenizer.tokenize(UpperCamelCase__		)
												self.assertListEqual(UpperCamelCase__					,	UpperCamelCase__		)

												# Testing conversion to ids without special tokens
												SCREAMING_SNAKE_CASE							:					Tuple           =							tokenizer.encode(UpperCamelCase__					,	add_special_tokens=UpperCamelCase__					,	add_prefix_space=UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Any           =							rust_tokenizer.encode(UpperCamelCase__					,	add_special_tokens=UpperCamelCase__		)
												self.assertListEqual(UpperCamelCase__					,	UpperCamelCase__		)

												# Testing conversion to ids with special tokens
												SCREAMING_SNAKE_CASE							:					List[Any]           =							self.get_rust_tokenizer(add_prefix_space=UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					List[Any]           =							tokenizer.encode(UpperCamelCase__					,	add_prefix_space=UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							rust_tokenizer.encode(UpperCamelCase__		)
												self.assertListEqual(UpperCamelCase__					,	UpperCamelCase__		)

												# Testing the unknown token
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokens + [rust_tokenizer.unk_token]
												SCREAMING_SNAKE_CASE							:					List[str]           =							[14, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCamelCase__		)					,	UpperCamelCase__		)


						def  __A ( self			:       List[Any]					,	*UpperCamelCase__			:       List[str]					,	**UpperCamelCase__			:       int		):





												'''simple docstring'''





												pass


						def  __A ( self			:       Tuple					,	UpperCamelCase__			:       Union[str, Any]=15		):





												'''simple docstring'''





												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
																								SCREAMING_SNAKE_CASE							:					Dict           =							self.rust_tokenizer_class.from_pretrained(UpperCamelCase__					,	**UpperCamelCase__		)

																								# Simple input
																								SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							'''This is a simple input'''
																								SCREAMING_SNAKE_CASE							:					Tuple           =							['''This is a simple input 1''', '''This is a simple input 2''']
																								SCREAMING_SNAKE_CASE							:					List[Any]           =							('''This is a simple input''', '''This is a pair''')
																								SCREAMING_SNAKE_CASE							:					Any           =							[
																								    ('''This is a simple input 1''', '''This is a simple input 2'''),
																								    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																								]

																								# Simple input tests
																								self.assertRaises(UpperCamelCase__					,	tokenizer_r.encode					,	UpperCamelCase__					,	max_length=UpperCamelCase__					,	padding='''max_length'''		)

																								# Simple input
																								self.assertRaises(UpperCamelCase__					,	tokenizer_r.encode_plus					,	UpperCamelCase__					,	max_length=UpperCamelCase__					,	padding='''max_length'''		)

																								# Simple input
																								self.assertRaises(
																								    UpperCamelCase__					,	tokenizer_r.batch_encode_plus					,	UpperCamelCase__					,	max_length=UpperCamelCase__					,	padding='''max_length'''					,	)

																								# Pair input
																								self.assertRaises(UpperCamelCase__					,	tokenizer_r.encode					,	UpperCamelCase__					,	max_length=UpperCamelCase__					,	padding='''max_length'''		)

																								# Pair input
																								self.assertRaises(UpperCamelCase__					,	tokenizer_r.encode_plus					,	UpperCamelCase__					,	max_length=UpperCamelCase__					,	padding='''max_length'''		)

																								# Pair input
																								self.assertRaises(
																								    UpperCamelCase__					,	tokenizer_r.batch_encode_plus					,	UpperCamelCase__					,	max_length=UpperCamelCase__					,	padding='''max_length'''					,	)


						def  __A ( self			:       Tuple		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					int           =							GPTaTokenizer.from_pretrained(self.tmpdirname					,	pad_token='''<pad>'''		)

												# Simple input
												SCREAMING_SNAKE_CASE							:					List[Any]           =							'''This is a simple input'''
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							['''This is a simple input looooooooong''', '''This is a simple input''']
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							('''This is a simple input''', '''This is a pair''')
												SCREAMING_SNAKE_CASE							:					List[str]           =							[
												    ('''This is a simple input loooooong''', '''This is a simple input'''),
												    ('''This is a simple pair loooooong''', '''This is a simple pair'''),
												]

												SCREAMING_SNAKE_CASE							:					List[str]           =							tokenizer.pad_token_id

												SCREAMING_SNAKE_CASE							:					Dict           =							tokenizer(UpperCamelCase__					,	padding='''max_length'''					,	max_length=30					,	return_tensors='''np'''		)
												SCREAMING_SNAKE_CASE							:					List[Any]           =							tokenizer(UpperCamelCase__					,	padding=UpperCamelCase__					,	truncate=UpperCamelCase__					,	return_tensors='''np'''		)
												SCREAMING_SNAKE_CASE							:					Dict           =							tokenizer(*UpperCamelCase__					,	padding='''max_length'''					,	max_length=60					,	return_tensors='''np'''		)
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokenizer(UpperCamelCase__					,	padding=UpperCamelCase__					,	truncate=UpperCamelCase__					,	return_tensors='''np'''		)

												# s
												# test single string max_length padding
												self.assertEqual(out_s['''input_ids'''].shape[-1]					,	30		)
												self.assertTrue(pad_token_id in out_s['''input_ids''']		)
												self.assertTrue(0 in out_s['''attention_mask''']		)

												# s2
												# test automatic padding
												self.assertEqual(out_sa['''input_ids'''].shape[-1]					,	33		)
												# long slice doesn't have padding
												self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]		)
												self.assertFalse(0 in out_sa['''attention_mask'''][0]		)
												# short slice does have padding
												self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]		)
												self.assertTrue(0 in out_sa['''attention_mask'''][1]		)

												# p
												# test single pair max_length padding
												self.assertEqual(out_p['''input_ids'''].shape[-1]					,	60		)
												self.assertTrue(pad_token_id in out_p['''input_ids''']		)
												self.assertTrue(0 in out_p['''attention_mask''']		)

												# p2
												# test automatic padding pair
												self.assertEqual(out_pa['''input_ids'''].shape[-1]					,	52		)
												# long slice pair doesn't have padding
												self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]		)
												self.assertFalse(0 in out_pa['''attention_mask'''][0]		)
												# short slice pair does have padding
												self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]		)
												self.assertTrue(0 in out_pa['''attention_mask'''][1]		)


						def  __A ( self			:       Optional[int]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Tuple           =							'''$$$'''
												SCREAMING_SNAKE_CASE							:					Dict           =							GPTaTokenizer.from_pretrained(self.tmpdirname					,	bos_token=UpperCamelCase__					,	add_bos_token=UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					List[Any]           =							'''This is a simple input'''
												SCREAMING_SNAKE_CASE							:					Dict           =							['''This is a simple input 1''', '''This is a simple input 2''']

												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokenizer.bos_token_id

												SCREAMING_SNAKE_CASE							:					Dict           =							tokenizer(UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					str           =							tokenizer(UpperCamelCase__		)

												self.assertEqual(out_s.input_ids[0]					,	UpperCamelCase__		)
												self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids		)		)

												SCREAMING_SNAKE_CASE							:					Any           =							tokenizer.decode(out_s.input_ids		)
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							tokenizer.batch_decode(out_sa.input_ids		)

												self.assertEqual(decode_s.split()[0]					,	UpperCamelCase__		)
												self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa		)		)


						def  __A ( self			:       Union[str, Any]		):





												'''simple docstring'''





												pass



						def  __A ( self			:       Dict		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							[self.get_tokenizer(do_lower_case=UpperCamelCase__					,	add_bos_token=UpperCamelCase__		)]
												for tokenizer in tokenizers:
																		with self.subTest(f"""{tokenizer.__class__.__name__}"""		):
																								SCREAMING_SNAKE_CASE							:					List[Any]           =							'''Encode this.'''
																								SCREAMING_SNAKE_CASE							:					Dict           =							'''This one too please.'''
																								SCREAMING_SNAKE_CASE							:					Any           =							tokenizer.encode(UpperCamelCase__					,	add_special_tokens=UpperCamelCase__		)
																								encoded_sequence += tokenizer.encode(UpperCamelCase__					,	add_special_tokens=UpperCamelCase__		)
																								SCREAMING_SNAKE_CASE							:					str           =							tokenizer.encode_plus(
																								    UpperCamelCase__					,	UpperCamelCase__					,	add_special_tokens=UpperCamelCase__					,	return_special_tokens_mask=UpperCamelCase__					,	)
																								SCREAMING_SNAKE_CASE							:					str           =							encoded_sequence_dict['''input_ids''']
																								SCREAMING_SNAKE_CASE							:					Dict           =							encoded_sequence_dict['''special_tokens_mask''']
																								self.assertEqual(len(UpperCamelCase__		)					,	len(UpperCamelCase__		)		)

																								SCREAMING_SNAKE_CASE							:					int           =							[
																								    (x if not special_tokens_mask[i] else None) for i, x in enumerate(UpperCamelCase__		)
																								]
																								SCREAMING_SNAKE_CASE							:					str           =							[x for x in filtered_sequence if x is not None]
																								self.assertEqual(UpperCamelCase__					,	UpperCamelCase__		)






@require_tokenizers
class       lowercase__		(     unittest.TestCase):


						def  __A ( self			:       Dict		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Tuple           =							AutoTokenizer.from_pretrained('''facebook/opt-350m'''					,	from_slow=UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Any           =							'''A photo of a cat'''

												SCREAMING_SNAKE_CASE							:					Any           =							tokenizer.encode(
												    UpperCamelCase__					,	)
												self.assertEqual(UpperCamelCase__					,	[2, 250, 1345, 9, 10, 4758]		)
												tokenizer.save_pretrained('''test_opt'''		)

												SCREAMING_SNAKE_CASE							:					Dict           =							AutoTokenizer.from_pretrained('''./test_opt'''		)
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							tokenizer.encode(
												    UpperCamelCase__					,	)
												self.assertEqual(UpperCamelCase__					,	[2, 250, 1345, 9, 10, 4758]		)


						def  __A ( self			:       Any		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Dict           =							AutoTokenizer.from_pretrained('''facebook/opt-350m'''					,	use_slow=UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							'''A photo of a cat'''

												SCREAMING_SNAKE_CASE							:					Tuple           =							tokenizer.encode(
												    UpperCamelCase__					,	)
												# Same as above
												self.assertEqual(UpperCamelCase__					,	[2, 250, 1345, 9, 10, 4758]		)



						@unittest.skip('''This test is failing because of a bug in the fast tokenizer'''		)
						def  __A ( self			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Tuple           =							AutoTokenizer.from_pretrained('''facebook/opt-350m'''					,	from_slow=UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					Any           =							'''bos'''
												SCREAMING_SNAKE_CASE							:					int           =							tokenizer.get_vocab()['''bos''']

												SCREAMING_SNAKE_CASE							:					Any           =							'''A photo of a cat'''
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							tokenizer.encode(
												    UpperCamelCase__					,	)
												# We changed the bos token
												self.assertEqual(UpperCamelCase__					,	[3_1957, 250, 1345, 9, 10, 4758]		)
												tokenizer.save_pretrained('''./tok'''		)
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							AutoTokenizer.from_pretrained('''./tok'''		)
												self.assertTrue(tokenizer.is_fast		)
												SCREAMING_SNAKE_CASE							:					List[Any]           =							tokenizer.encode(
												    UpperCamelCase__					,	)
												self.assertEqual(UpperCamelCase__					,	[3_1957, 250, 1345, 9, 10, 4758]		)






import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class       lowercase__		(     UpperCamelCase_			,							unittest.TestCase):
						UpperCamelCase_  =	FunnelTokenizer
						UpperCamelCase_  =	FunnelTokenizerFast
						UpperCamelCase_  =	True
						UpperCamelCase_  =	True


						def  __A ( self			:       Union[str, Any]		):





												'''simple docstring'''





												super().setUp()

												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							[
												    '''<unk>''',
												    '''<cls>''',
												    '''<sep>''',
												    '''want''',
												    '''##want''',
												    '''##ed''',
												    '''wa''',
												    '''un''',
												    '''runn''',
												    '''##ing''',
												    ''',''',
												    '''low''',
												    '''lowest''',
												]
												SCREAMING_SNAKE_CASE							:					List[str]           =							os.path.join(self.tmpdirname					,	VOCAB_FILES_NAMES['''vocab_file''']		)
												with open(self.vocab_file					,	'''w'''					,	encoding='''utf-8'''		) as vocab_writer:
																		vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)


						def  __A ( self			:       int					,	**UpperCamelCase__			:       List[Any]		):





												'''simple docstring'''





												return FunnelTokenizer.from_pretrained(self.tmpdirname					,	**UpperCamelCase__		)


						def  __A ( self			:       int					,	**UpperCamelCase__			:       List[Any]		):





												'''simple docstring'''





												return FunnelTokenizerFast.from_pretrained(self.tmpdirname					,	**UpperCamelCase__		)


						def  __A ( self			:       Any					,	UpperCamelCase__			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							'''UNwant\u00E9d,running'''
												SCREAMING_SNAKE_CASE							:					int           =							'''unwanted, running'''
												return input_text, output_text


						def  __A ( self			:       Any		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					List[Any]           =							self.tokenizer_class(self.vocab_file		)

												SCREAMING_SNAKE_CASE							:					int           =							tokenizer.tokenize('''UNwant\u00E9d,running'''		)
												self.assertListEqual(UpperCamelCase__					,	['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']		)
												self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__		)					,	[7, 4, 5, 10, 8, 9]		)



						def  __A ( self			:       str		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Dict           =							self.get_tokenizers(do_lower_case=UpperCamelCase__		)
												for tokenizer in tokenizers:
																		SCREAMING_SNAKE_CASE							:					int           =							tokenizer('''UNwant\u00E9d,running'''		)
																		SCREAMING_SNAKE_CASE							:					Optional[Any]           =							len(inputs['''input_ids''']		) - 1
																		self.assertListEqual(inputs['''token_type_ids''']					,	[2] + [0] * sentence_len		)

																		SCREAMING_SNAKE_CASE							:					List[Any]           =							tokenizer('''UNwant\u00E9d,running'''					,	'''UNwant\u00E9d,running'''		)
																		self.assertListEqual(inputs['''token_type_ids''']					,	[2] + [0] * sentence_len + [1] * sentence_len		)





