


'''simple docstring'''

import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class   snake_case					(					lowercase    ,		unittest.TestCase    ):





  """simple docstring"""




  _lowerCamelCase						=						None
  _lowerCamelCase						=						BloomTokenizerFast
  _lowerCamelCase						=						BloomTokenizerFast
  _lowerCamelCase						=						True
  _lowerCamelCase						=						False
  _lowerCamelCase						=						"tokenizer_file"
  _lowerCamelCase						=						{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}






  def 						snake_case	(					self  ):







   """simple docstring"""
   super().setUp()
   lowerCamelCase_  				=						BloomTokenizerFast.from_pretrained("bigscience/tokenizer"  )
   tokenizer.save_pretrained(self.tmpdirname  )






  def 						snake_case	(					self    ,						**UpperCamelCase  ):







   """simple docstring"""
   kwargs.update(self.special_tokens_map  )
   return BloomTokenizerFast.from_pretrained(self.tmpdirname    ,						**UpperCamelCase  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.get_rust_tokenizer()

   lowerCamelCase_  				=						["The quick brown fox</s>", "jumps over the lazy dog</s>"]
   lowerCamelCase_  				=						[[2175, 2_3714, 7_3173, 14_4252, 2], [77, 13_2619, 3478, 368, 10_9586, 3_5433, 2]]

   lowerCamelCase_  				=						tokenizer.batch_encode_plus(UpperCamelCase  )["input_ids"]
   self.assertListEqual(UpperCamelCase    ,						UpperCamelCase  )

   lowerCamelCase_  				=						tokenizer.batch_decode(UpperCamelCase  )
   self.assertListEqual(UpperCamelCase    ,						UpperCamelCase  )






  def 						snake_case	(					self    ,						UpperCamelCase=6  ):







   """simple docstring"""
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
     lowerCamelCase_  				=						self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,						**UpperCamelCase  )
     # tokenizer_r.pad_token = None # Hotfixing padding = None
     # Simple input
     lowerCamelCase_  				=						"This is a simple input"
     lowerCamelCase_  				=						["This is a simple input 1", "This is a simple input 2"]
     lowerCamelCase_  				=						("This is a simple input", "This is a pair")
     lowerCamelCase_  				=						[
         ("This is a simple input 1", "This is a simple input 2"),
         ("This is a simple pair 1", "This is a simple pair 2"),
     ]

     # Simple input tests
     try:
      tokenizer_r.encode(UpperCamelCase    ,						max_length=UpperCamelCase  )
      tokenizer_r.encode_plus(UpperCamelCase    ,						max_length=UpperCamelCase  )

      tokenizer_r.batch_encode_plus(UpperCamelCase    ,						max_length=UpperCamelCase  )
      tokenizer_r.encode(UpperCamelCase    ,						max_length=UpperCamelCase  )
      tokenizer_r.batch_encode_plus(UpperCamelCase    ,						max_length=UpperCamelCase  )
     except ValueError:
      self.fail("Bloom Tokenizer should be able to deal with padding"  )

     lowerCamelCase_  				=						None  # Hotfixing padding = None
     self.assertRaises(UpperCamelCase    ,						tokenizer_r.encode    ,						UpperCamelCase    ,						max_length=UpperCamelCase    ,						padding="max_length"  )

     # Simple input
     self.assertRaises(UpperCamelCase    ,						tokenizer_r.encode_plus    ,						UpperCamelCase    ,						max_length=UpperCamelCase    ,						padding="max_length"  )

     # Simple input
     self.assertRaises(
         UpperCamelCase    ,						tokenizer_r.batch_encode_plus    ,						UpperCamelCase    ,						max_length=UpperCamelCase    ,						padding="max_length"    ,						)

     # Pair input
     self.assertRaises(UpperCamelCase    ,						tokenizer_r.encode    ,						UpperCamelCase    ,						max_length=UpperCamelCase    ,						padding="max_length"  )

     # Pair input
     self.assertRaises(UpperCamelCase    ,						tokenizer_r.encode_plus    ,						UpperCamelCase    ,						max_length=UpperCamelCase    ,						padding="max_length"  )

     # Pair input
     self.assertRaises(
         UpperCamelCase    ,						tokenizer_r.batch_encode_plus    ,						UpperCamelCase    ,						max_length=UpperCamelCase    ,						padding="max_length"    ,						)






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.get_rust_tokenizer()
   lowerCamelCase_  				=						load_dataset("xnli"    ,						"all_languages"    ,						split="test"    ,						streaming=UpperCamelCase  )

   lowerCamelCase_  				=						next(iter(UpperCamelCase  )  )["premise"]  # pick up one data
   lowerCamelCase_  				=						list(sample_data.values()  )

   lowerCamelCase_  				=						list(map(tokenizer.encode    ,						UpperCamelCase  )  )
   lowerCamelCase_  				=						[tokenizer.decode(UpperCamelCase    ,						clean_up_tokenization_spaces=UpperCamelCase  ) for x in output_tokens]
   self.assertListEqual(UpperCamelCase    ,						UpperCamelCase  )







  def 						snake_case	(					self  ):







   """simple docstring"""
   # The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
   # any sequence length constraints. This test of the parent class will fail since it relies on the
   # maximum sequence length of the positoonal embeddings.
   self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  )    ,						1  )
   self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  )    ,						1  )







'''simple docstring'''

from __future__ import annotations





def __snake_case      (   UpperCAmelCase_		:		int      ,       UpperCAmelCase_		:		int	):
 if b == 0:
  return (1, 0)
 ((lowerCamelCase_)				,(lowerCamelCase_))  				=						extended_euclid(UpperCAmelCase_      ,       a % b	)
 lowerCamelCase_  				=						a // b
 return (y, x - k * y)





def __snake_case      (   UpperCAmelCase_		:		int      ,       UpperCAmelCase_		:		int      ,       UpperCAmelCase_		:		int      ,       UpperCAmelCase_		:		int	):
 ((lowerCamelCase_)				,(lowerCamelCase_))  				=						extended_euclid(UpperCAmelCase_      ,       UpperCAmelCase_	)
 lowerCamelCase_  				=						na * na
 lowerCamelCase_  				=						ra * x * na + ra * y * na
 return (n % m + m) % m





def __snake_case      (   UpperCAmelCase_		:		int      ,       UpperCAmelCase_		:		int	):
 ((lowerCamelCase_)				,(lowerCamelCase_))  				=						extended_euclid(UpperCAmelCase_      ,       UpperCAmelCase_	)
 if b < 0:
  lowerCamelCase_  				=						(b % n + n) % n
 return b





def __snake_case      (   UpperCAmelCase_		:		int      ,       UpperCAmelCase_		:		int      ,       UpperCAmelCase_		:		int      ,       UpperCAmelCase_		:		int	):
 lowerCamelCase_				,lowerCamelCase_  				=						invert_modulo(UpperCAmelCase_      ,       UpperCAmelCase_	), invert_modulo(UpperCAmelCase_      ,       UpperCAmelCase_	)
 lowerCamelCase_  				=						na * na
 lowerCamelCase_  				=						ra * x * na + ra * y * na
 return (n % m + m) % m


if __name__ == "__main__":
     from doctest import testmod

     testmod(name="""chinese_remainder_theorem""", verbose=True)
     testmod(name="""chinese_remainder_theorem2""", verbose=True)
     testmod(name="""invert_modulo""", verbose=True)
     testmod(name="""extended_euclid""", verbose=True)



