'''simple docstring'''




import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 					__magic_name__		(					_UpperCAmelCase,  unittest.TestCase):
			UpperCamelCase__       = GPTaTokenizer
			UpperCamelCase__       = GPTaTokenizerFast
			UpperCamelCase__       = True
			UpperCamelCase__       = {'''add_prefix_space''': True}
			UpperCamelCase__       = False







			def 			SCREAMING_SNAKE_CASE_  (			self       :						Tuple					):
										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										lowercase_      :    Optional[Any]       		= [
										    """l""",
										    """o""",
										    """w""",
										    """e""",
										    """r""",
										    """s""",
										    """t""",
										    """i""",
										    """d""",
										    """n""",
										    """\u0120""",
										    """\u0120l""",
										    """\u0120n""",
										    """\u0120lo""",
										    """\u0120low""",
										    """er""",
										    """\u0120lowest""",
										    """\u0120newer""",
										    """\u0120wider""",
										    """<unk>""",
										    """<|endoftext|>""",
										]
										lowercase_      :    Dict       		= dict(zip(lowercase_       ,    range(len(lowercase_					)					)					)					)
										lowercase_      :    List[Any]       		= ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
										lowercase_      :    Tuple       		= {"""unk_token""": """<unk>"""}

										lowercase_      :    Dict       		= os.path.join(self.tmpdirname       ,    VOCAB_FILES_NAMES["""vocab_file"""]					)
										lowercase_      :    Any       		= os.path.join(self.tmpdirname       ,    VOCAB_FILES_NAMES["""merges_file"""]					)
										with open(self.vocab_file       ,    """w"""       ,    encoding="""utf-8"""					) as fp:
																	fp.write(json.dumps(lowercase_					) + """\n"""					)
										with open(self.merges_file       ,    """w"""       ,    encoding="""utf-8"""					) as fp:
																	fp.write("""\n""".join(lowercase_					)					)







			def 			SCREAMING_SNAKE_CASE_  (			self       :						Dict       ,    **lowercase_       :						int					):
										kwargs.update(self.special_tokens_map					)
										return GPTaTokenizer.from_pretrained(self.tmpdirname       ,    **lowercase_					)







			def 			SCREAMING_SNAKE_CASE_  (			self       :						Tuple       ,    **lowercase_       :						List[Any]					):
										kwargs.update(self.special_tokens_map					)
										return GPTaTokenizerFast.from_pretrained(self.tmpdirname       ,    **lowercase_					)







			def 			SCREAMING_SNAKE_CASE_  (			self       :						Union[str, Any]       ,    lowercase_       :						Any					):
										lowercase_      :    Dict       		= """lower newer"""
										lowercase_      :    int       		= """lower newer"""
										return input_text, output_text







			def 			SCREAMING_SNAKE_CASE_  (			self       :						Union[str, Any]					):
										lowercase_      :    List[str]       		= GPTaTokenizer(self.vocab_file       ,    self.merges_file       ,    **self.special_tokens_map					)
										lowercase_      :    Union[str, Any]       		= """lower newer"""
										lowercase_      :    List[str]       		= ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
										lowercase_      :    Optional[int]       		= tokenizer.tokenize(lowercase_       ,    add_prefix_space=lowercase_					)
										self.assertListEqual(lowercase_       ,    lowercase_					)

										lowercase_      :    Optional[Any]       		= tokens + [tokenizer.unk_token]
										lowercase_      :    Dict       		= [14, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_					)       ,    lowercase_					)







			def 			SCREAMING_SNAKE_CASE_  (			self       :						Optional[Any]					):
										if not self.test_rust_tokenizer:
																	return

										lowercase_      :    Tuple       		= self.get_tokenizer()
										lowercase_      :    List[str]       		= self.get_rust_tokenizer(add_prefix_space=lowercase_					)

										lowercase_      :    List[str]       		= """lower newer"""

										# Testing tokenization
										lowercase_      :    Dict       		= tokenizer.tokenize(lowercase_       ,    add_prefix_space=lowercase_					)
										lowercase_      :    int       		= rust_tokenizer.tokenize(lowercase_					)
										self.assertListEqual(lowercase_       ,    lowercase_					)

										# Testing conversion to ids without special tokens
										lowercase_      :    Tuple       		= tokenizer.encode(lowercase_       ,    add_special_tokens=lowercase_       ,    add_prefix_space=lowercase_					)
										lowercase_      :    Optional[Any]       		= rust_tokenizer.encode(lowercase_       ,    add_special_tokens=lowercase_					)
										self.assertListEqual(lowercase_       ,    lowercase_					)

										# Testing conversion to ids with special tokens
										lowercase_      :    Dict       		= self.get_rust_tokenizer(add_prefix_space=lowercase_					)
										lowercase_      :    List[str]       		= tokenizer.encode(lowercase_       ,    add_prefix_space=lowercase_					)
										lowercase_      :    Dict       		= rust_tokenizer.encode(lowercase_					)
										self.assertListEqual(lowercase_       ,    lowercase_					)

										# Testing the unknown token
										lowercase_      :    str       		= tokens + [rust_tokenizer.unk_token]
										lowercase_      :    Any       		= [14, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowercase_					)       ,    lowercase_					)







			def 			SCREAMING_SNAKE_CASE_  (			self       :						Tuple       ,    *lowercase_       :						str       ,    **lowercase_       :						Optional[int]					):
										# It's very difficult to mix/test pretokenization with byte-level
										# And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
										pass







			def 			SCREAMING_SNAKE_CASE_  (			self       :						List[Any]       ,    lowercase_       :						Optional[Any]=15					):
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																								lowercase_      :    List[str]       		= self.rust_tokenizer_class.from_pretrained(lowercase_       ,    **lowercase_					)

																								# Simple input
																								lowercase_      :    Dict       		= """This is a simple input"""
																								lowercase_      :    Union[str, Any]       		= ["""This is a simple input 1""", """This is a simple input 2"""]
																								lowercase_      :    Tuple       		= ("""This is a simple input""", """This is a pair""")
																								lowercase_      :    Any       		= [
																								    ("""This is a simple input 1""", """This is a simple input 2"""),
																								    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																								]

																								# Simple input tests
																								self.assertRaises(lowercase_       ,    tokenizer_r.encode       ,    lowercase_       ,    max_length=lowercase_       ,    padding="""max_length"""					)

																								# Simple input
																								self.assertRaises(lowercase_       ,    tokenizer_r.encode_plus       ,    lowercase_       ,    max_length=lowercase_       ,    padding="""max_length"""					)

																								# Simple input
																								self.assertRaises(
																								    lowercase_       ,    tokenizer_r.batch_encode_plus       ,    lowercase_       ,    max_length=lowercase_       ,    padding="""max_length"""       ,    )

																								# Pair input
																								self.assertRaises(lowercase_       ,    tokenizer_r.encode       ,    lowercase_       ,    max_length=lowercase_       ,    padding="""max_length"""					)

																								# Pair input
																								self.assertRaises(lowercase_       ,    tokenizer_r.encode_plus       ,    lowercase_       ,    max_length=lowercase_       ,    padding="""max_length"""					)

																								# Pair input
																								self.assertRaises(
																								    lowercase_       ,    tokenizer_r.batch_encode_plus       ,    lowercase_       ,    max_length=lowercase_       ,    padding="""max_length"""       ,    )







			def 			SCREAMING_SNAKE_CASE_  (			self       :						str					):
										lowercase_      :    Optional[int]       		= GPTaTokenizer.from_pretrained(self.tmpdirname       ,    pad_token="""<pad>"""					)

										# Simple input
										lowercase_      :    str       		= """This is a simple input"""
										lowercase_      :    str       		= ["""This is a simple input looooooooong""", """This is a simple input"""]
										lowercase_      :    Optional[int]       		= ("""This is a simple input""", """This is a pair""")
										lowercase_      :    Dict       		= [
										    ("""This is a simple input loooooong""", """This is a simple input"""),
										    ("""This is a simple pair loooooong""", """This is a simple pair"""),
										]

										lowercase_      :    int       		= tokenizer.pad_token_id

										lowercase_      :    Any       		= tokenizer(lowercase_       ,    padding="""max_length"""       ,    max_length=30       ,    return_tensors="""np"""					)
										lowercase_      :    int       		= tokenizer(lowercase_       ,    padding=lowercase_       ,    truncate=lowercase_       ,    return_tensors="""np"""					)
										lowercase_      :    str       		= tokenizer(*lowercase_       ,    padding="""max_length"""       ,    max_length=60       ,    return_tensors="""np"""					)
										lowercase_      :    Optional[Any]       		= tokenizer(lowercase_       ,    padding=lowercase_       ,    truncate=lowercase_       ,    return_tensors="""np"""					)

										# s
										# test single string max_length padding
										self.assertEqual(out_s["""input_ids"""].shape[-1]       ,    30					)
										self.assertTrue(pad_token_id in out_s["""input_ids"""]					)
										self.assertTrue(0 in out_s["""attention_mask"""]					)

										# s2
										# test automatic padding
										self.assertEqual(out_sa["""input_ids"""].shape[-1]       ,    33					)
										# long slice doesn't have padding
										self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]					)
										self.assertFalse(0 in out_sa["""attention_mask"""][0]					)
										# short slice does have padding
										self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]					)
										self.assertTrue(0 in out_sa["""attention_mask"""][1]					)

										# p
										# test single pair max_length padding
										self.assertEqual(out_p["""input_ids"""].shape[-1]       ,    60					)
										self.assertTrue(pad_token_id in out_p["""input_ids"""]					)
										self.assertTrue(0 in out_p["""attention_mask"""]					)

										# p2
										# test automatic padding pair
										self.assertEqual(out_pa["""input_ids"""].shape[-1]       ,    52					)
										# long slice pair doesn't have padding
										self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]					)
										self.assertFalse(0 in out_pa["""attention_mask"""][0]					)
										# short slice pair does have padding
										self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]					)
										self.assertTrue(0 in out_pa["""attention_mask"""][1]					)







			def 			SCREAMING_SNAKE_CASE_  (			self       :						List[Any]					):
										lowercase_      :    Tuple       		= """$$$"""
										lowercase_      :    Tuple       		= GPTaTokenizer.from_pretrained(self.tmpdirname       ,    bos_token=lowercase_       ,    add_bos_token=lowercase_					)

										lowercase_      :    List[Any]       		= """This is a simple input"""
										lowercase_      :    str       		= ["""This is a simple input 1""", """This is a simple input 2"""]

										lowercase_      :    int       		= tokenizer.bos_token_id

										lowercase_      :    Dict       		= tokenizer(lowercase_					)
										lowercase_      :    Tuple       		= tokenizer(lowercase_					)

										self.assertEqual(out_s.input_ids[0]       ,    lowercase_					)
										self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids					)					)

										lowercase_      :    List[Any]       		= tokenizer.decode(out_s.input_ids					)
										lowercase_      :    Dict       		= tokenizer.batch_decode(out_sa.input_ids					)

										self.assertEqual(decode_s.split()[0]       ,    lowercase_					)
										self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa					)					)







			def 			SCREAMING_SNAKE_CASE_  (			self       :						Optional[int]					):
										pass







			def 			SCREAMING_SNAKE_CASE_  (			self       :						str					):
										# TODO: change to self.get_tokenizers() when the fast version is implemented
										lowercase_      :    Tuple       		= [self.get_tokenizer(do_lower_case=lowercase_       ,    add_bos_token=lowercase_					)]
										for tokenizer in tokenizers:
																	with self.subTest(f'''{tokenizer.__class__.__name__}'''					):
																								lowercase_      :    int       		= """Encode this."""
																								lowercase_      :    Tuple       		= """This one too please."""
																								lowercase_      :    List[str]       		= tokenizer.encode(lowercase_       ,    add_special_tokens=lowercase_					)
																								encoded_sequence += tokenizer.encode(lowercase_       ,    add_special_tokens=lowercase_					)
																								lowercase_      :    List[Any]       		= tokenizer.encode_plus(
																								    lowercase_       ,    lowercase_       ,    add_special_tokens=lowercase_       ,    return_special_tokens_mask=lowercase_       ,    )
																								lowercase_      :    int       		= encoded_sequence_dict["""input_ids"""]
																								lowercase_      :    List[str]       		= encoded_sequence_dict["""special_tokens_mask"""]
																								self.assertEqual(len(lowercase_					)       ,    len(lowercase_					)					)

																								lowercase_      :    int       		= [
																								    (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowercase_					)
																								]
																								lowercase_      :    List[str]       		= [x for x in filtered_sequence if x is not None]
																								self.assertEqual(lowercase_       ,    lowercase_					)





@require_tokenizers
class 					__magic_name__		(					unittest.TestCase):







			def 			SCREAMING_SNAKE_CASE_  (			self       :						List[str]					):
										# More context:
										# https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
										# https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
										# https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

										lowercase_      :    str       		= AutoTokenizer.from_pretrained("""facebook/opt-350m"""       ,    from_slow=lowercase_					)
										lowercase_      :    Optional[int]       		= """A photo of a cat"""

										lowercase_      :    Any       		= tokenizer.encode(
										    lowercase_       ,    )
										self.assertEqual(lowercase_       ,    [2, 250, 1345, 9, 10, 4758]					)
										tokenizer.save_pretrained("""test_opt"""					)

										lowercase_      :    Optional[Any]       		= AutoTokenizer.from_pretrained("""./test_opt"""					)
										lowercase_      :    List[Any]       		= tokenizer.encode(
										    lowercase_       ,    )
										self.assertEqual(lowercase_       ,    [2, 250, 1345, 9, 10, 4758]					)







			def 			SCREAMING_SNAKE_CASE_  (			self       :						Dict					):
										lowercase_      :    int       		= AutoTokenizer.from_pretrained("""facebook/opt-350m"""       ,    use_slow=lowercase_					)
										lowercase_      :    List[Any]       		= """A photo of a cat"""

										lowercase_      :    List[Any]       		= tokenizer.encode(
										    lowercase_       ,    )
										# Same as above
										self.assertEqual(lowercase_       ,    [2, 250, 1345, 9, 10, 4758]					)







			@unittest.skip("""This test is failing because of a bug in the fast tokenizer"""					)
			def 			SCREAMING_SNAKE_CASE_  (			self       :						Tuple					):
										lowercase_      :    Any       		= AutoTokenizer.from_pretrained("""facebook/opt-350m"""       ,    from_slow=lowercase_					)

										lowercase_      :    Union[str, Any]       		= """bos"""
										lowercase_      :    int       		= tokenizer.get_vocab()["""bos"""]

										lowercase_      :    Union[str, Any]       		= """A photo of a cat"""
										lowercase_      :    Dict       		= tokenizer.encode(
										    lowercase_       ,    )
										# We changed the bos token
										self.assertEqual(lowercase_       ,    [31957, 250, 1345, 9, 10, 4758]					)
										tokenizer.save_pretrained("""./tok"""					)
										lowercase_      :    Optional[int]       		= AutoTokenizer.from_pretrained("""./tok"""					)
										self.assertTrue(tokenizer.is_fast					)
										lowercase_      :    Tuple       		= tokenizer.encode(
										    lowercase_       ,    )
										self.assertEqual(lowercase_       ,    [31957, 250, 1345, 9, 10, 4758]					)






'''simple docstring'''




_lowercase		:				int	       =		"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"




def 							lowerCamelCase			(   UpperCAmelCase__ :      bytes				)    ->   bytes:
							# Make sure the supplied data is a bytes-like object
							if not isinstance(UpperCAmelCase__   ,   UpperCAmelCase__				):
														lowercase_      :    Union[str, Any]       		= F'''a bytes-like object is required, not \'{data.__class__.__name__}\''''
														raise TypeError(UpperCAmelCase__				)

							lowercase_      :    Dict       		= """""".join(bin(UpperCAmelCase__				)[2:].zfill(8				) for byte in data				)

							lowercase_      :    Union[str, Any]       		= len(UpperCAmelCase__				) % 6 != 0

							if padding_needed:
														# The padding that will be added later
														lowercase_      :    List[Any]       		= b"""=""" * ((6 - len(UpperCAmelCase__				) % 6) // 2)

														# Append binary_stream with arbitrary binary digits (0's by default) to make its
														# length a multiple of 6.
														binary_stream += "0" * (6 - len(UpperCAmelCase__				) % 6)
							else:
														lowercase_      :    Union[str, Any]       		= b""""""

							# Encode every 6 binary digits to their corresponding Base64 character
							return (
							    "".join(
							        B64_CHARSET[int(binary_stream[index : index + 6]   ,   2				)]
							        for index in range(0   ,   len(UpperCAmelCase__				)   ,   6				)				).encode()
							    + padding
							)




def 							lowerCamelCase			(   UpperCAmelCase__ :      str				)    ->   bytes:
							# Make sure encoded_data is either a string or a bytes-like object
							if not isinstance(UpperCAmelCase__   ,   UpperCAmelCase__				) and not isinstance(UpperCAmelCase__   ,   UpperCAmelCase__				):
														lowercase_      :    List[str]       		= (
														    """argument should be a bytes-like object or ASCII string, """
														    F'''not \'{encoded_data.__class__.__name__}\''''
														)
														raise TypeError(UpperCAmelCase__				)

							# In case encoded_data is a bytes-like object, make sure it contains only
							# ASCII characters so we convert it to a string object
							if isinstance(UpperCAmelCase__   ,   UpperCAmelCase__				):
														try:
																					lowercase_      :    Optional[int]       		= encoded_data.decode("""utf-8"""				)
														except UnicodeDecodeError:
																					raise ValueError("""base64 encoded data should only contain ASCII characters"""				)

							lowercase_      :    Any       		= encoded_data.count("""="""				)

							# Check if the encoded string contains non base64 characters
							if padding:
														assert all(
														    char in B64_CHARSET for char in encoded_data[:-padding]				), "Invalid base64 character(s) found."
							else:
														assert all(
														    char in B64_CHARSET for char in encoded_data				), "Invalid base64 character(s) found."

							# Check the padding
							assert len(UpperCAmelCase__				) % 4 == 0 and padding < 3, "Incorrect padding"

							if padding:
														# Remove padding if there is one
														lowercase_      :    Optional[int]       		= encoded_data[:-padding]

														lowercase_      :    Any       		= """""".join(
														    bin(B64_CHARSET.index(UpperCAmelCase__				)				)[2:].zfill(6				) for char in encoded_data				)[: -padding * 2]
							else:
														lowercase_      :    int       		= """""".join(
														    bin(B64_CHARSET.index(UpperCAmelCase__				)				)[2:].zfill(6				) for char in encoded_data				)

							lowercase_      :    Optional[int]       		= [
							    int(binary_stream[index : index + 8]   ,   2				)
							    for index in range(0   ,   len(UpperCAmelCase__				)   ,   8				)
							]

							return bytes(UpperCAmelCase__				)


if __name__ == "__main__":
						import doctest

						doctest.testmod()





