'''simple docstring'''

import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__     :	List[str]				=			get_tests_dir('fixtures/test_sentencepiece.model')


@require_sentencepiece
@require_tokenizers
class 				_lowercase	(      lowerCAmelCase		,unittest.TestCase   ):
      '''simple docstring'''


      UpperCAmelCase_      :      List[Any]      							=   XLNetTokenizer
      UpperCAmelCase_      :      Optional[Any]      							=   XLNetTokenizerFast
      UpperCAmelCase_      :      Union[str, Any]      							=   True
      UpperCAmelCase_      :      List[Any]      							=   True




      def 		lowerCAmelCase__					(				self							)			->			Union[str, Any]:



         '''simple docstring'''



         super().setUp()

         # We have a SentencePiece fixture for testing
         UpperCAmelCase__							:				List[str]   				=  XLNetTokenizer(lowerCamelCase_					,keep_accents=lowerCamelCase_							)
         tokenizer.sanitize_special_tokens()
         tokenizer.save_pretrained(self.tmpdirname							)




      def 		lowerCAmelCase__					(				self							)			->			str:



         '''simple docstring'''



         UpperCAmelCase__							:				Tuple   				=  '''<s>'''
         UpperCAmelCase__							:				Optional[Any]   				=  1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase_							)					,lowerCamelCase_							)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase_							)					,lowerCamelCase_							)




      def 		lowerCAmelCase__					(				self							)			->			str:



         '''simple docstring'''



         UpperCAmelCase__							:				str   				=  list(self.get_tokenizer().get_vocab().keys()							)

         self.assertEqual(vocab_keys[0]					,'''<unk>'''							)
         self.assertEqual(vocab_keys[1]					,'''<s>'''							)
         self.assertEqual(vocab_keys[-1]					,'''<eod>'''							)
         self.assertEqual(len(lowerCamelCase_							)					,1006							)




      def 		lowerCAmelCase__					(				self							)			->			str:



         '''simple docstring'''



         self.assertEqual(self.get_tokenizer().vocab_size					,1000							)




      def 		lowerCAmelCase__					(				self							)			->			str:



         '''simple docstring'''



         UpperCAmelCase__							:				Any   				=  XLNetTokenizer(lowerCamelCase_					,keep_accents=lowerCamelCase_							)

         UpperCAmelCase__							:				List[str]   				=  tokenizer.tokenize('''This is a test'''							)
         self.assertListEqual(lowerCamelCase_					,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

         self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_							)					,[285, 46, 10, 170, 382]							)

         UpperCAmelCase__							:				Optional[Any]   				=  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
         self.assertListEqual(
             lowerCamelCase_					,[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''9''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''é''',
                 '''.''',
             ]					,)
         UpperCAmelCase__							:				int   				=  tokenizer.convert_tokens_to_ids(lowerCamelCase_							)
         self.assertListEqual(lowerCamelCase_					,[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]							)

         UpperCAmelCase__							:				List[Any]   				=  tokenizer.convert_ids_to_tokens(lowerCamelCase_							)
         self.assertListEqual(
             lowerCamelCase_					,[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''<unk>''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''<unk>''',
                 '''.''',
             ]					,)




      def 		lowerCAmelCase__					(				self							)			->			Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__							:				Optional[int]   				=  XLNetTokenizer(lowerCamelCase_					,do_lower_case=lowerCamelCase_							)
         UpperCAmelCase__							:				List[str]   				=  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
         self.assertListEqual(
             lowerCamelCase_					,[
                 SPIECE_UNDERLINE + '''''',
                 '''i''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''9''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''se''',
                 '''.''',
             ]					,)
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,['''▁he''', '''ll''', '''o''']							)




      def 		lowerCAmelCase__					(				self							)			->			Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__							:				str   				=  XLNetTokenizer(lowerCamelCase_					,do_lower_case=lowerCamelCase_							)
         UpperCAmelCase__							:				str   				=  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
         self.assertListEqual(
             lowerCamelCase_					,[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''9''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''se''',
                 '''.''',
             ]					,)




      @slow
      def 		lowerCAmelCase__					(				self							)			->			List[Any]:



         '''simple docstring'''



         UpperCAmelCase__							:				List[Any]   				=  XLNetTokenizer.from_pretrained('''xlnet-base-cased'''							)

         UpperCAmelCase__							:				int   				=  tokenizer.encode('''sequence builders'''					,add_special_tokens=lowerCamelCase_							)
         UpperCAmelCase__							:				Tuple   				=  tokenizer.encode('''multi-sequence build'''					,add_special_tokens=lowerCamelCase_							)

         UpperCAmelCase__							:				Optional[Any]   				=  tokenizer.build_inputs_with_special_tokens(lowerCamelCase_							)
         UpperCAmelCase__							:				Tuple   				=  tokenizer.build_inputs_with_special_tokens(lowerCamelCase_					,lowerCamelCase_							)

         assert encoded_sentence == text + [4, 3]
         assert encoded_pair == text + [4] + text_a + [4, 3]




      @slow
      def 		lowerCAmelCase__					(				self							)			->			List[str]:



         '''simple docstring'''



         UpperCAmelCase__							:				Optional[int]   				=  {'''input_ids''': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=lowerCamelCase_					,model_name='''xlnet-base-cased'''					,revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''					,)





'''simple docstring'''

from __future__ import annotations

import sys
from collections import deque
from typing import Generic, TypeVar

UpperCamelCase__     :	Tuple				=			TypeVar('T')


class 				_lowercase	(      Generic[T]   ):
      '''simple docstring'''


      UpperCAmelCase_      :      deque[T]  # Cache store of keys
      UpperCAmelCase_      :      set[T]  # References of the keys in cache
      UpperCAmelCase_      :      int      							=   10  # Maximum capacity of cache




      def __init__(				self					,lowerCamelCase_							)			->			None:



         '''simple docstring'''



         UpperCAmelCase__							:				Any   				=  deque()
         UpperCAmelCase__							:				Optional[int]   				=  set()
         if not n:
            UpperCAmelCase__							:				List[Any]   				=  sys.maxsize
         elif n < 0:
            raise ValueError('''n should be an integer greater than 0.'''							)
         else:
            UpperCAmelCase__							:				Any   				=  n




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_							)			->			None:



         '''simple docstring'''



         if x not in self.key_reference:
            if len(self.dq_store							) == LRUCache._MAX_CAPACITY:
               UpperCAmelCase__							:				int   				=  self.dq_store.pop()
               self.key_reference.remove(lowerCamelCase_							)
         else:
            self.dq_store.remove(lowerCamelCase_							)

         self.dq_store.appendleft(lowerCamelCase_							)
         self.key_reference.add(lowerCamelCase_							)




      def 		lowerCAmelCase__					(				self							)			->			None:



         '''simple docstring'''



         for k in self.dq_store:
            print(lowerCamelCase_							)




      def __repr__(				self							)			->			str:



         '''simple docstring'''



         return f'''LRUCache({self._MAX_CAPACITY}) => {list(self.dq_store							)}'''


if __name__ == "__main__":
  import doctest

  doctest.testmod()

  UpperCamelCase__     :	LRUCache[str | int]				=			LRUCache(4)
  lru_cache.refer('A')
  lru_cache.refer(2)
  lru_cache.refer(3)
  lru_cache.refer('A')
  lru_cache.refer(4)
  lru_cache.refer(5)
  lru_cache.display()

  print(lru_cache)
  assert str(lru_cache) == "LRUCache(4) => [5, 4, 'A', 3]"




