'''simple docstring'''

import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__     :	str				=			get_tests_dir('fixtures/test_sentencepiece.model')


@require_sentencepiece
@require_tokenizers
class 				_lowercase	(      __SCREAMING_SNAKE_CASE		,unittest.TestCase   ):
 '''simple docstring'''


 UpperCAmelCase_      :      Optional[int]      							=   XLMRobertaTokenizer
 UpperCAmelCase_      :      List[str]      							=   XLMRobertaTokenizerFast
 UpperCAmelCase_      :      List[Any]      							=   True
 UpperCAmelCase_      :      List[str]      							=   True




 def 		lowerCAmelCase__					(				self							)			->			Optional[int]:



    '''simple docstring'''



    super().setUp()

    # We have a SentencePiece fixture for testing
    UpperCAmelCase__							:				List[Any]   				=  XLMRobertaTokenizer(__snake_case					,keep_accents=__snake_case							)
    tokenizer.save_pretrained(self.tmpdirname							)




 def 		lowerCAmelCase__					(				self							)			->			List[Any]:



    '''simple docstring'''



    UpperCAmelCase__							:				List[str]   				=  '''<pad>'''
    UpperCAmelCase__							:				Optional[int]   				=  1

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case							)					,__snake_case							)
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case							)					,__snake_case							)




 def 		lowerCAmelCase__					(				self							)			->			List[Any]:



    '''simple docstring'''



    UpperCAmelCase__							:				int   				=  list(self.get_tokenizer().get_vocab().keys()							)

    self.assertEqual(vocab_keys[0]					,'''<s>'''							)
    self.assertEqual(vocab_keys[1]					,'''<pad>'''							)
    self.assertEqual(vocab_keys[-1]					,'''<mask>'''							)
    self.assertEqual(len(__snake_case							)					,1002							)




 def 		lowerCAmelCase__					(				self							)			->			Union[str, Any]:



    '''simple docstring'''



    self.assertEqual(self.get_tokenizer().vocab_size					,1002							)




 def 		lowerCAmelCase__					(				self							)			->			Optional[int]:



    '''simple docstring'''



    UpperCAmelCase__							:				str   				=  XLMRobertaTokenizer(__snake_case					,keep_accents=__snake_case							)

    UpperCAmelCase__							:				int   				=  tokenizer.tokenize('''This is a test'''							)
    self.assertListEqual(__snake_case					,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(__snake_case							)					,[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					,)

    UpperCAmelCase__							:				str   				=  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
    self.assertListEqual(
        __snake_case					,[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''9''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''é''',
            '''.''',
        ]					,)
    UpperCAmelCase__							:				Tuple   				=  tokenizer.convert_tokens_to_ids(__snake_case							)
    self.assertListEqual(
        __snake_case					,[
            value + tokenizer.fairseq_offset
            for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
        ]					,)

    UpperCAmelCase__							:				Any   				=  tokenizer.convert_ids_to_tokens(__snake_case							)
    self.assertListEqual(
        __snake_case					,[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''<unk>''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''<unk>''',
            '''.''',
        ]					,)




 def 		lowerCAmelCase__					(				self							)			->			Tuple:



    '''simple docstring'''



    if not self.test_slow_tokenizer:
       # as we don't have a slow version, we can't compare the outputs between slow and fast versions
       return

    UpperCAmelCase__							:				int   				=  (self.rust_tokenizer_class, '''hf-internal-testing/tiny-xlm-roberta''', {})
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
          UpperCAmelCase__							:				List[str]   				=  self.rust_tokenizer_class.from_pretrained(__snake_case					,**__snake_case							)
          UpperCAmelCase__							:				int   				=  self.tokenizer_class.from_pretrained(__snake_case					,**__snake_case							)

          UpperCAmelCase__							:				Optional[int]   				=  tempfile.mkdtemp()

          UpperCAmelCase__							:				List[Any]   				=  tokenizer_r.save_pretrained(__snake_case							)
          UpperCAmelCase__							:				List[str]   				=  tokenizer_p.save_pretrained(__snake_case							)

          # Checks it save with the same files + the tokenizer.json file for the fast one
          self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)
          UpperCAmelCase__							:				Any   				=  tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f							)
          self.assertSequenceEqual(__snake_case					,__snake_case							)

          # Checks everything loads correctly in the same way
          UpperCAmelCase__							:				Dict   				=  tokenizer_r.from_pretrained(__snake_case							)
          UpperCAmelCase__							:				Optional[Any]   				=  tokenizer_p.from_pretrained(__snake_case							)

          # Check special tokens are set accordingly on Rust and Python
          for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(__snake_case					,__snake_case							)							)
             # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
             # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

          shutil.rmtree(__snake_case							)

          # Save tokenizer rust, legacy_format=True
          UpperCAmelCase__							:				List[str]   				=  tempfile.mkdtemp()

          UpperCAmelCase__							:				Optional[Any]   				=  tokenizer_r.save_pretrained(__snake_case					,legacy_format=__snake_case							)
          UpperCAmelCase__							:				Any   				=  tokenizer_p.save_pretrained(__snake_case							)

          # Checks it save with the same files
          self.assertSequenceEqual(__snake_case					,__snake_case							)

          # Checks everything loads correctly in the same way
          UpperCAmelCase__							:				Any   				=  tokenizer_r.from_pretrained(__snake_case							)
          UpperCAmelCase__							:				Optional[int]   				=  tokenizer_p.from_pretrained(__snake_case							)

          # Check special tokens are set accordingly on Rust and Python
          for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(__snake_case					,__snake_case							)							)

          shutil.rmtree(__snake_case							)

          # Save tokenizer rust, legacy_format=False
          UpperCAmelCase__							:				Dict   				=  tempfile.mkdtemp()

          UpperCAmelCase__							:				Optional[int]   				=  tokenizer_r.save_pretrained(__snake_case					,legacy_format=__snake_case							)
          UpperCAmelCase__							:				Optional[int]   				=  tokenizer_p.save_pretrained(__snake_case							)

          # Checks it saved the tokenizer.json file
          self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)

          # Checks everything loads correctly in the same way
          UpperCAmelCase__							:				Tuple   				=  tokenizer_r.from_pretrained(__snake_case							)
          UpperCAmelCase__							:				str   				=  tokenizer_p.from_pretrained(__snake_case							)

          # Check special tokens are set accordingly on Rust and Python
          for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(__snake_case					,__snake_case							)							)

          shutil.rmtree(__snake_case							)




 @cached_property
 def 		lowerCAmelCase__					(				self							)			->			Any:



    '''simple docstring'''



    return XLMRobertaTokenizer.from_pretrained('''xlm-roberta-base'''							)




 def 		lowerCAmelCase__					(				self							)			->			List[str]:



    '''simple docstring'''



    with tempfile.NamedTemporaryFile() as f:
       shutil.copyfile(__snake_case					,f.name							)
       UpperCAmelCase__							:				Optional[int]   				=  XLMRobertaTokenizer(f.name					,keep_accents=__snake_case							)
       UpperCAmelCase__							:				Any   				=  pickle.dumps(__snake_case							)
    pickle.loads(__snake_case							)




 def 		lowerCAmelCase__					(				self							)			->			Optional[Any]:



    '''simple docstring'''



    if not self.test_rust_tokenizer:
       return

    UpperCAmelCase__							:				Optional[Any]   				=  self.get_tokenizer()
    UpperCAmelCase__							:				Tuple   				=  self.get_rust_tokenizer()

    UpperCAmelCase__							:				List[str]   				=  '''I was born in 92000, and this is falsé.'''

    UpperCAmelCase__							:				Any   				=  tokenizer.tokenize(__snake_case							)
    UpperCAmelCase__							:				Any   				=  rust_tokenizer.tokenize(__snake_case							)
    self.assertListEqual(__snake_case					,__snake_case							)

    UpperCAmelCase__							:				List[str]   				=  tokenizer.encode(__snake_case					,add_special_tokens=__snake_case							)
    UpperCAmelCase__							:				Dict   				=  rust_tokenizer.encode(__snake_case					,add_special_tokens=__snake_case							)
    self.assertListEqual(__snake_case					,__snake_case							)

    UpperCAmelCase__							:				Optional[Any]   				=  self.get_rust_tokenizer()
    UpperCAmelCase__							:				List[str]   				=  tokenizer.encode(__snake_case							)
    UpperCAmelCase__							:				Dict   				=  rust_tokenizer.encode(__snake_case							)
    self.assertListEqual(__snake_case					,__snake_case							)




 @slow
 def 		lowerCAmelCase__					(				self							)			->			List[str]:



    '''simple docstring'''



    UpperCAmelCase__							:				Tuple   				=  '''Hello World!'''
    UpperCAmelCase__							:				Dict   				=  [0, 35378, 6661, 38, 2]
    # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
    # xlmr.eval()
    # xlmr.encode(symbols)

    self.assertListEqual(__snake_case					,self.big_tokenizer.encode(__snake_case							)							)




 @slow
 def 		lowerCAmelCase__					(				self							)			->			Dict:



    '''simple docstring'''



    UpperCAmelCase__							:				Any   				=  (
        '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
        ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
    )
    UpperCAmelCase__							:				Optional[int]   				=  [
        0,
        3293,
        83,
        10,
        4552,
        4989,
        7986,
        678,
        10,
        5915,
        111,
        179459,
        124850,
        4,
        6044,
        237,
        12,
        6,
        5,
        6,
        4,
        6780,
        705,
        15,
        1388,
        44,
        378,
        10114,
        711,
        152,
        20,
        6,
        5,
        22376,
        642,
        1221,
        15190,
        34153,
        450,
        5608,
        959,
        1119,
        57702,
        136,
        186,
        47,
        1098,
        29367,
        47,
        # 4426, # What fairseq tokenizes from "<unk>": "_<"
        # 3678, # What fairseq tokenizes from "<unk>": "unk"
        # 2740, # What fairseq tokenizes from "<unk>": ">"
        3,  # What we tokenize from "<unk>": "<unk>"
        6,  # Residue from the tokenization: an extra sentencepiece underline
        4,
        6044,
        237,
        6284,
        50901,
        528,
        31,
        90,
        34,
        927,
        2,
    ]
    # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
    # xlmr.eval()
    # xlmr.encode(symbols)

    self.assertListEqual(__snake_case					,self.big_tokenizer.encode(__snake_case							)							)




 @slow
 def 		lowerCAmelCase__					(				self							)			->			Any:



    '''simple docstring'''



    UpperCAmelCase__							:				Any   				=  {'''input_ids''': [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
    # fmt: on

    self.tokenizer_integration_test_util(
        expected_encoding=__snake_case					,model_name='''xlm-roberta-base'''					,revision='''d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3'''					,)





'''simple docstring'''

import math







def __UpperCamelCase(   _A					:				float  ,			_A					:				float    ):
   '''simple docstring'''
   return math.pow(_A  ,			2    ) - a







def __UpperCamelCase(   _A					:				float    ):
   '''simple docstring'''
   return 2 * x







def __UpperCamelCase(   _A					:				float    ):
   '''simple docstring'''
   UpperCAmelCase__							:				int   				=  2.0

   while start <= a:
      UpperCAmelCase__							:				int   				=  math.pow(_A  ,			2    )

   return start







def __UpperCamelCase(   _A					:				float  ,			_A					:				int = 99_99  ,			_A					:				float = 0.0_0_0_0_0_0_0_0_0_0_0_0_0_1    ):
   '''simple docstring'''

   if a < 0:
      raise ValueError('''math domain error'''    )

   UpperCAmelCase__							:				Any   				=  get_initial_point(_A    )

   for _ in range(_A    ):
      UpperCAmelCase__							:				Optional[Any]   				=  value
      UpperCAmelCase__							:				Tuple   				=  value - fx(_A  ,			_A    ) / fx_derivative(_A    )
      if abs(prev_value - value    ) < tolerance:
         return value

   return value


if __name__ == "__main__":
  from doctest import testmod

  testmod()




