

'''simple docstring'''

import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 							_snake_case (       _a	,  unittest.TestCase						):
   _A       :     List[Any]       =					None
   _A       :     Tuple       =					BloomTokenizerFast
   _A       :     List[Any]       =					BloomTokenizerFast
   _A       :     List[Any]       =					True
   _A       :     Tuple       =					False
   _A       :     List[str]       =					'''tokenizer_file'''
   _A       :     int       =					{'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}
   def     __UpperCamelCase						( self			:      List[str]							):
     super().setUp()
     SCREAMING_SNAKE_CASE:str						      =					BloomTokenizerFast.from_pretrained("bigscience/tokenizer"							)
     tokenizer.save_pretrained(self.tmpdirname							)
   def     __UpperCamelCase						( self			:      List[Any]     ,**SCREAMING_SNAKE_CASE__			:      Union[str, Any]							):
     kwargs.update(self.special_tokens_map							)
     return BloomTokenizerFast.from_pretrained(self.tmpdirname     ,**SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      Optional[int]							):
     SCREAMING_SNAKE_CASE:int						      =					self.get_rust_tokenizer()

     SCREAMING_SNAKE_CASE:Optional[int]						      =					["The quick brown fox</s>", "jumps over the lazy dog</s>"]
     SCREAMING_SNAKE_CASE:str						      =					[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

     SCREAMING_SNAKE_CASE:Any						      =					tokenizer.batch_encode_plus(SCREAMING_SNAKE_CASE__							)["input_ids"]
     self.assertListEqual(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:Dict						      =					tokenizer.batch_decode(SCREAMING_SNAKE_CASE__							)
     self.assertListEqual(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      List[Any]     ,SCREAMING_SNAKE_CASE__			:      List[Any]=6							):
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
         SCREAMING_SNAKE_CASE:str						      =					self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__     ,**SCREAMING_SNAKE_CASE__							)
         # tokenizer_r.pad_token = None # Hotfixing padding = None
         # Simple input
         SCREAMING_SNAKE_CASE:Optional[Any]						      =					"This is a simple input"
         SCREAMING_SNAKE_CASE:List[str]						      =					["This is a simple input 1", "This is a simple input 2"]
         SCREAMING_SNAKE_CASE:int						      =					("This is a simple input", "This is a pair")
         SCREAMING_SNAKE_CASE:Union[str, Any]						      =					[
             ("This is a simple input 1", "This is a simple input 2"),
             ("This is a simple pair 1", "This is a simple pair 2"),
         ]

         # Simple input tests
         try:
           tokenizer_r.encode(SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__							)
           tokenizer_r.encode_plus(SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__							)

           tokenizer_r.batch_encode_plus(SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__							)
           tokenizer_r.encode(SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__							)
           tokenizer_r.batch_encode_plus(SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__							)
         except ValueError:
           self.fail("Bloom Tokenizer should be able to deal with padding"							)

         SCREAMING_SNAKE_CASE:Optional[Any]						      =					None  # Hotfixing padding = None
         self.assertRaises(SCREAMING_SNAKE_CASE__     ,tokenizer_r.encode     ,SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__     ,padding="max_length"							)

         # Simple input
         self.assertRaises(SCREAMING_SNAKE_CASE__     ,tokenizer_r.encode_plus     ,SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__     ,padding="max_length"							)

         # Simple input
         self.assertRaises(
             SCREAMING_SNAKE_CASE__     ,tokenizer_r.batch_encode_plus     ,SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__     ,padding="max_length"     ,)

         # Pair input
         self.assertRaises(SCREAMING_SNAKE_CASE__     ,tokenizer_r.encode     ,SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__     ,padding="max_length"							)

         # Pair input
         self.assertRaises(SCREAMING_SNAKE_CASE__     ,tokenizer_r.encode_plus     ,SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__     ,padding="max_length"							)

         # Pair input
         self.assertRaises(
             SCREAMING_SNAKE_CASE__     ,tokenizer_r.batch_encode_plus     ,SCREAMING_SNAKE_CASE__     ,max_length=SCREAMING_SNAKE_CASE__     ,padding="max_length"     ,)
   def     __UpperCamelCase						( self			:      Dict							):
     SCREAMING_SNAKE_CASE:List[Any]						      =					self.get_rust_tokenizer()
     SCREAMING_SNAKE_CASE:Any						      =					load_dataset("xnli"     ,"all_languages"     ,split="test"     ,streaming=SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:List[str]						      =					next(iter(SCREAMING_SNAKE_CASE__							)							)["premise"]  # pick up one data
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					list(sample_data.values()							)

     SCREAMING_SNAKE_CASE:Tuple						      =					list(map(tokenizer.encode     ,SCREAMING_SNAKE_CASE__							)							)
     SCREAMING_SNAKE_CASE:str						      =					[tokenizer.decode(SCREAMING_SNAKE_CASE__     ,clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__							) for x in output_tokens]
     self.assertListEqual(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)







   def     __UpperCamelCase						( self			:      int							):
     # The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
     # any sequence length constraints. This test of the parent class will fail since it relies on the
     # maximum sequence length of the positoonal embeddings.
     self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map							)     ,1							)
     self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()							)[0]							)     ,1							)









'''simple docstring'''

from importlib import import_module

from .logging import get_logger


A_       						=							get_logger(__name__)





class 							_snake_case :
   def __init__( self			:      Dict     ,SCREAMING_SNAKE_CASE__			:      Any     ,SCREAMING_SNAKE_CASE__			:      Dict=None							):
     SCREAMING_SNAKE_CASE:Any						      =					attrs or []
     if module is not None:
       for key in module.__dict__:
         if key in attrs or not key.startswith("__"							):
           setattr(self     ,SCREAMING_SNAKE_CASE__     ,getattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)							)
     SCREAMING_SNAKE_CASE:Any						      =					module._original_module if isinstance(SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj							) else module





class 							_snake_case :
   _A       :     Any       =					[]
   def __init__( self			:      Any     ,SCREAMING_SNAKE_CASE__			:      List[str]     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]=None							):
     SCREAMING_SNAKE_CASE:Any						      =					obj
     SCREAMING_SNAKE_CASE:Any						      =					target
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					new
     SCREAMING_SNAKE_CASE:Any						      =					target.split("."							)[0]
     SCREAMING_SNAKE_CASE:str						      =					{}
     SCREAMING_SNAKE_CASE:List[Any]						      =					attrs or []
   def __enter__( self			:      Tuple							):
     *SCREAMING_SNAKE_CASE      ,    SCREAMING_SNAKE_CASE:int						      =					self.target.split("."							)

     # Patch modules:
     # it's used to patch attributes of submodules like "os.path.join";
     # in this case we need to patch "os" and "os.path"

     for i in range(len(SCREAMING_SNAKE_CASE__							)							):
       try:
         SCREAMING_SNAKE_CASE:int						      =					import_module(".".join(submodules[: i + 1]							)							)
       except ModuleNotFoundError:
         continue
       # We iterate over all the globals in self.obj in case we find "os" or "os.path"
       for attr in self.obj.__dir__():
         SCREAMING_SNAKE_CASE:str						      =					getattr(self.obj     ,SCREAMING_SNAKE_CASE__							)
         # We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
         # This allows to patch renamed modules like "from os import path as ospath".
         if obj_attr is submodule or (
             (isinstance(SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj							) and obj_attr._original_module is submodule)
         ):
           SCREAMING_SNAKE_CASE:Union[str, Any]						      =					obj_attr
           # patch at top level
           setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj(SCREAMING_SNAKE_CASE__     ,attrs=self.attrs							)							)
           SCREAMING_SNAKE_CASE:int						      =					getattr(self.obj     ,SCREAMING_SNAKE_CASE__							)
           # construct lower levels patches
           for key in submodules[i + 1 :]:
             setattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj(getattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)     ,attrs=self.attrs							)							)
             SCREAMING_SNAKE_CASE:Dict						      =					getattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)
           # finally set the target attribute
           setattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,self.new							)

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

     if submodules:  # if it's an attribute of a submodule like "os.path.join"
       try:
         SCREAMING_SNAKE_CASE:Any						      =					getattr(import_module(".".join(SCREAMING_SNAKE_CASE__							)							)     ,SCREAMING_SNAKE_CASE__							)
       except (AttributeError, ModuleNotFoundError):
         return
       # We iterate over all the globals in self.obj in case we find "os.path.join"
       for attr in self.obj.__dir__():
         # We don't check for the name of the global, but rather if its value *is* "os.path.join".
         # This allows to patch renamed attributes like "from os.path import join as pjoin".
         if getattr(self.obj     ,SCREAMING_SNAKE_CASE__							) is attr_value:
           SCREAMING_SNAKE_CASE:Union[str, Any]						      =					getattr(self.obj     ,SCREAMING_SNAKE_CASE__							)
           setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,self.new							)
     elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
       SCREAMING_SNAKE_CASE:int						      =					globals()["__builtins__"][target_attr]
       setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,self.new							)
     else:
       raise RuntimeError(F'''Tried to patch attribute {target_attr} instead of a submodule.'''							)
   def __exit__( self			:      List[Any]     ,*SCREAMING_SNAKE_CASE__			:      Union[str, Any]							):
     for attr in list(self.original							):
       setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,self.original.pop(SCREAMING_SNAKE_CASE__							)							)
   def     __UpperCamelCase						( self			:      Optional[int]							):
     self.__enter__()
     self._active_patches.append(self							)







   def     __UpperCamelCase						( self			:      Dict							):
     try:
       self._active_patches.remove(self							)
     except ValueError:
       # If the patch hasn't been started this will fail
       return None

     return self.__exit__()






