





"""simple docstring"""






import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 	lowerCAmelCase					(      snake_case			,	unittest.TestCase							):
							lowerCAmelCase__						 =  None
							lowerCAmelCase__						 =  BloomTokenizerFast
							lowerCAmelCase__						 =  BloomTokenizerFast
							lowerCAmelCase__						 =  True
							lowerCAmelCase__						 =  False
							lowerCAmelCase__						 =  """tokenizer_file"""
							lowerCAmelCase__						 =  {"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}






							def 							__A (   self     ):
														super().setUp()
														_UpperCAmelCase     =     BloomTokenizerFast.from_pretrained('bigscience/tokenizer'     )
														tokenizer.save_pretrained(self.tmpdirname     )






							def 							__A (   self    ,  **a__     ):
														kwargs.update(self.special_tokens_map     )
														return BloomTokenizerFast.from_pretrained(self.tmpdirname    ,  **a__     )






							def 							__A (   self     ):
														_UpperCAmelCase     =     self.get_rust_tokenizer()

														_UpperCAmelCase     =     ['The quick brown fox</s>', 'jumps over the lazy dog</s>']
														_UpperCAmelCase     =     [[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

														_UpperCAmelCase     =     tokenizer.batch_encode_plus(a__     )['input_ids']
														self.assertListEqual(a__    ,  a__     )

														_UpperCAmelCase     =     tokenizer.batch_decode(a__     )
														self.assertListEqual(a__    ,  a__     )






							def 							__A (   self    ,  a__=6     ):
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
																												_UpperCAmelCase     =     self.rust_tokenizer_class.from_pretrained(a__    ,  **a__     )
																												# tokenizer_r.pad_token = None # Hotfixing padding = None
																												# Simple input
																												_UpperCAmelCase     =     'This is a simple input'
																												_UpperCAmelCase     =     ['This is a simple input 1', 'This is a simple input 2']
																												_UpperCAmelCase     =     ('This is a simple input', 'This is a pair')
																												_UpperCAmelCase     =     [
																												    ('This is a simple input 1', 'This is a simple input 2'),
																												    ('This is a simple pair 1', 'This is a simple pair 2'),
																												]

																												# Simple input tests
																												try:
																																			tokenizer_r.encode(a__    ,  max_length=a__     )
																																			tokenizer_r.encode_plus(a__    ,  max_length=a__     )

																																			tokenizer_r.batch_encode_plus(a__    ,  max_length=a__     )
																																			tokenizer_r.encode(a__    ,  max_length=a__     )
																																			tokenizer_r.batch_encode_plus(a__    ,  max_length=a__     )
																												except ValueError:
																																			self.fail('Bloom Tokenizer should be able to deal with padding'     )

																												_UpperCAmelCase     =     None  # Hotfixing padding = None
																												self.assertRaises(a__    ,  tokenizer_r.encode    ,  a__    ,  max_length=a__    ,  padding='max_length'     )

																												# Simple input
																												self.assertRaises(a__    ,  tokenizer_r.encode_plus    ,  a__    ,  max_length=a__    ,  padding='max_length'     )

																												# Simple input
																												self.assertRaises(
																												    a__    ,  tokenizer_r.batch_encode_plus    ,  a__    ,  max_length=a__    ,  padding='max_length'    ,  )

																												# Pair input
																												self.assertRaises(a__    ,  tokenizer_r.encode    ,  a__    ,  max_length=a__    ,  padding='max_length'     )

																												# Pair input
																												self.assertRaises(a__    ,  tokenizer_r.encode_plus    ,  a__    ,  max_length=a__    ,  padding='max_length'     )

																												# Pair input
																												self.assertRaises(
																												    a__    ,  tokenizer_r.batch_encode_plus    ,  a__    ,  max_length=a__    ,  padding='max_length'    ,  )






							def 							__A (   self     ):
														_UpperCAmelCase     =     self.get_rust_tokenizer()
														_UpperCAmelCase     =     load_dataset('xnli'    ,  'all_languages'    ,  split='test'    ,  streaming=a__     )

														_UpperCAmelCase     =     next(iter(a__     )     )['premise']  # pick up one data
														_UpperCAmelCase     =     list(sample_data.values()     )

														_UpperCAmelCase     =     list(map(tokenizer.encode    ,  a__     )     )
														_UpperCAmelCase     =     [tokenizer.decode(a__    ,  clean_up_tokenization_spaces=a__     ) for x in output_tokens]
														self.assertListEqual(a__    ,  a__     )






							def 							__A (   self     ):
														# The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
														# any sequence length constraints. This test of the parent class will fail since it relies on the
														# maximum sequence length of the positoonal embeddings.
														self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map     )    ,  1     )
														self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()     )[0]     )    ,  1     )













"""simple docstring"""






def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       int:






       """simple docstring"""




       return number | (1 << position)

def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       int:






       """simple docstring"""




       return number & ~(1 << position)

def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       int:






       """simple docstring"""




       return number ^ (1 << position)

def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       bool:






       """simple docstring"""




       return ((number >> position) & 1) == 1

def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       int:






       """simple docstring"""




       return int((number & (1 << position)) != 0							)


if __name__ == "__main__":
    import doctest

    doctest.testmod()






