






"""simple docstring"""





import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_	:				str										= get_tests_dir("""fixtures/test_sentencepiece.model""")


@require_sentencepiece
@require_tokenizers
class      _lowercase (     lowerCAmelCase		,  unittest.TestCase						):
      _a   :     List[str]								=   ReformerTokenizer
      _a   :     Dict								=   ReformerTokenizerFast
      _a   :     str								=   True
      _a   :     Optional[int]								=   False
      _a   :     Optional[Any]								=   True


      def _UpperCamelCase						(    self				:	str						):





            """simple docstring"""







            super().setUp()

            __snake_case      :    Optional[int]  			=ReformerTokenizer(a							,   keep_accents=a						)
            tokenizer.save_pretrained(self.tmpdirname						)


      def _UpperCamelCase						(    self				:	Union[str, Any]						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			='''<s>'''
            __snake_case      :    int  			=1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(a						)							,   a						)
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(a						)							,   a						)


      def _UpperCamelCase						(    self				:	Optional[int]						):





            """simple docstring"""







            __snake_case      :    Dict  			=list(self.get_tokenizer().get_vocab().keys()						)

            self.assertEqual(vocab_keys[0]							,   '''<unk>'''						)
            self.assertEqual(vocab_keys[1]							,   '''<s>'''						)
            self.assertEqual(vocab_keys[-1]							,   '''j'''						)
            self.assertEqual(len(a						)							,   1_0_0_0						)


      def _UpperCamelCase						(    self				:	Union[str, Any]						):





            """simple docstring"""







            self.assertEqual(self.get_tokenizer().vocab_size							,   1_0_0_0						)


      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            if not self.test_rust_tokenizer:
                  return

            __snake_case      :    List[str]  			=self.get_tokenizer()
            __snake_case      :    Optional[Any]  			=self.get_rust_tokenizer()

            __snake_case      :    List[Any]  			='''I was born in 92000, and this is falsé.'''

            __snake_case      :    List[Any]  			=tokenizer.tokenize(a						)
            __snake_case      :    str  			=rust_tokenizer.tokenize(a						)
            self.assertListEqual(a							,   a						)

            __snake_case      :    Optional[Any]  			=tokenizer.encode(a							,   add_special_tokens=a						)
            __snake_case      :    Union[str, Any]  			=rust_tokenizer.encode(a							,   add_special_tokens=a						)
            self.assertListEqual(a							,   a						)

            __snake_case      :    str  			=self.get_rust_tokenizer()
            __snake_case      :    Dict  			=tokenizer.encode(a						)
            __snake_case      :    List[str]  			=rust_tokenizer.encode(a						)
            self.assertListEqual(a							,   a						)


      def _UpperCamelCase						(    self				:	Dict							,   a				:	Optional[Any]=1_5						):





            """simple docstring"""







            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
                        __snake_case      :    List[str]  			=self.rust_tokenizer_class.from_pretrained(a							,   **a						)

                        # Simple input
                        __snake_case      :    Optional[int]  			='''This is a simple input'''
                        __snake_case      :    Optional[int]  			=['''This is a simple input 1''', '''This is a simple input 2''']
                        __snake_case      :    int  			=('''This is a simple input''', '''This is a pair''')
                        __snake_case      :    int  			=[
                            ('''This is a simple input 1''', '''This is a simple input 2'''),
                            ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                        ]

                        # Simple input tests
                        self.assertRaises(a							,   tokenizer_r.encode							,   a							,   max_length=a							,   padding='''max_length'''						)

                        # Simple input
                        self.assertRaises(a							,   tokenizer_r.encode_plus							,   a							,   max_length=a							,   padding='''max_length'''						)

                        # Simple input
                        self.assertRaises(
                            a							,   tokenizer_r.batch_encode_plus							,   a							,   max_length=a							,   padding='''max_length'''							,   )

                        # Pair input
                        self.assertRaises(a							,   tokenizer_r.encode							,   a							,   max_length=a							,   padding='''max_length'''						)

                        # Pair input
                        self.assertRaises(a							,   tokenizer_r.encode_plus							,   a							,   max_length=a							,   padding='''max_length'''						)

                        # Pair input
                        self.assertRaises(
                            a							,   tokenizer_r.batch_encode_plus							,   a							,   max_length=a							,   padding='''max_length'''							,   )


      def _UpperCamelCase						(    self				:	Optional[Any]						):





            """simple docstring"""







            pass


      def _UpperCamelCase						(    self				:	Optional[Any]						):





            """simple docstring"""







            __snake_case      :    str  			=ReformerTokenizer(a							,   keep_accents=a						)

            __snake_case      :    Tuple  			=tokenizer.tokenize('''This is a test'''						)
            self.assertListEqual(a							,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']						)

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(a						)							,   [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]							,   )

            __snake_case      :    Tuple  			=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''						)
            self.assertListEqual(
                a							,   [
                    SPIECE_UNDERLINE + '''I''',
                    SPIECE_UNDERLINE + '''was''',
                    SPIECE_UNDERLINE + '''b''',
                    '''or''',
                    '''n''',
                    SPIECE_UNDERLINE + '''in''',
                    SPIECE_UNDERLINE + '''''',
                    '''9''',
                    '''2''',
                    '''0''',
                    '''0''',
                    '''0''',
                    ''',''',
                    SPIECE_UNDERLINE + '''and''',
                    SPIECE_UNDERLINE + '''this''',
                    SPIECE_UNDERLINE + '''is''',
                    SPIECE_UNDERLINE + '''f''',
                    '''al''',
                    '''s''',
                    '''é''',
                    '''.''',
                ]							,   )
            __snake_case      :    str  			=tokenizer.convert_tokens_to_ids(a						)
            self.assertListEqual(
                a							,   [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]							,   )

            __snake_case      :    Union[str, Any]  			=tokenizer.convert_ids_to_tokens(a						)
            self.assertListEqual(
                a							,   [
                    SPIECE_UNDERLINE + '''I''',
                    SPIECE_UNDERLINE + '''was''',
                    SPIECE_UNDERLINE + '''b''',
                    '''or''',
                    '''n''',
                    SPIECE_UNDERLINE + '''in''',
                    SPIECE_UNDERLINE + '''''',
                    '''<unk>''',
                    '''2''',
                    '''0''',
                    '''0''',
                    '''0''',
                    ''',''',
                    SPIECE_UNDERLINE + '''and''',
                    SPIECE_UNDERLINE + '''this''',
                    SPIECE_UNDERLINE + '''is''',
                    SPIECE_UNDERLINE + '''f''',
                    '''al''',
                    '''s''',
                    '''<unk>''',
                    '''.''',
                ]							,   )


      @cached_property
      def _UpperCamelCase						(    self				:	List[Any]						):





            """simple docstring"""







            return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''						)


      @slow
      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case      :    List[Any]  			='''Hello World!'''
            __snake_case      :    List[Any]  			=[1_2_6, 3_2, 2_6_2, 1_5_2, 3_8, 7_2, 2_8_7]

            self.assertListEqual(a							,   self.big_tokenizer.encode(a						)						)


      @slow
      def _UpperCamelCase						(    self				:	int						):





            """simple docstring"""







            __snake_case      :    Tuple  			=(
                '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
                ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
            )
            __snake_case      :    List[Any]  			=[
                1_0_8,
                2_6_5,
                2_4,
                1_1_1,
                4,
                2_5_8,
                1_5_6,
                3_5,
                2_8,
                2_7_5,
                3,
                2_5_9,
                2_9_7,
                2_6_0,
                8_4,
                4,
                3_5,
                1_1_0,
                4_4,
                8,
                2_5_9,
                9_1,
                2_6_8,
                2_1,
                1_1,
                2_0_9,
                2_7_4,
                1_0_9,
                2_6_6,
                2_7_7,
                1_1_7,
                8_6,
                9_3,
                3_1_5,
                2_5_8,
                2_7_8,
                2_5_8,
                2_7_7,
                2_5_8,
                0,
                2_5_8,
                2_8_8,
                2_5_8,
                3_1_9,
                2_5_8,
                0,
                2_5_8,
                0,
                2_5_8,
                0,
                2_5_8,
                0,
                2_5_8,
                2_8_7,
                2_5_8,
                3_1_5,
                2_5_8,
                2_8_9,
                2_5_8,
                2_7_8,
                9_9,
                2_6_9,
                2_6_6,
                2_6_2,
                8,
                2_5_9,
                2_4_1,
                4,
                2_1_7,
                2_3_0,
                2_6_8,
                2_6_6,
                5_5,
                1_6_8,
                1_0_6,
                7_5,
                1_9_3,
                2_6_6,
                2_2_3,
                2_7,
                4_9,
                2_6,
                2_8_2,
                2_5,
                2_6_4,
                2_9_9,
                1_9,
                2_6,
                0,
                2_5_8,
                2_7_7,
                1_1_7,
                8_6,
                9_3,
                1_7_6,
                1_8_3,
                2_7_0,
                1_1,
                2_6_2,
                4_2,
                6_1,
                2_6_5,
            ]

            self.assertListEqual(a							,   self.big_tokenizer.encode(a						)						)


      @require_torch
      @slow
      def _UpperCamelCase						(    self				:	Optional[Any]						):





            """simple docstring"""







            import torch

            from transformers import ReformerConfig, ReformerModel

            # Build sequence
            __snake_case      :    Optional[int]  			=list(self.big_tokenizer.get_vocab().keys()						)[:1_0]
            __snake_case      :    Union[str, Any]  			=''' '''.join(a						)
            __snake_case      :    Optional[int]  			=self.big_tokenizer.encode_plus(a							,   return_tensors='''pt'''						)
            __snake_case      :    Optional[int]  			=self.big_tokenizer.batch_encode_plus([sequence, sequence]							,   return_tensors='''pt'''						)

            __snake_case      :    Optional[int]  			=ReformerConfig()
            # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
            __snake_case      :    Optional[Any]  			=encoded_sequence['''input_ids'''].shape
            __snake_case      :    Tuple  			=ReformerModel(a						)

            # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
            assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

            with torch.no_grad():
                  model(**a						)
                  model(**a						)







      @slow
      def _UpperCamelCase						(    self				:	List[Any]						):





            """simple docstring"""







            __snake_case      :    List[str]  			={'''input_ids''': [[1_0_8, 2_6_5, 2_4, 1_1_1, 4, 2_5_8, 1_5_6, 7, 5_1, 2_7_9, 5_8, 7, 7_6, 2_5, 6_9, 2_7_8], [1_4_0, 2_4_3, 2_6_4, 1_3_4, 1_7, 2_6_7, 7_7, 2_6_3, 2_2, 2_6_2, 2_9_7, 2_5_8, 3_0_4, 1_7_7, 2_7_9, 2_6_6, 1_4, 8_9, 1_3, 3_5, 2_6_1, 2_9_9, 2_7_2, 1_3_7, 2_7_5, 2_7_8]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
            # fmt: on

            # This tokenizer does not know some characters like ")".
            # That is the reason why we use very simple texts here.
            # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
            __snake_case      :    Union[str, Any]  			=[
                '''This is a very simple sentence.''',
                '''The quick brown fox jumps over the lazy dog.''',
            ]

            self.tokenizer_integration_test_util(
                expected_encoding=a							,   model_name='''google/reformer-crime-and-punishment'''							,   revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a'''							,   padding=a							,   sequences=a							,   )










"""simple docstring"""





import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class      _lowercase (     lowerCAmelCase		,  unittest.TestCase						):
      _a   :     List[str]								=   MgpstrTokenizer
      _a   :     int								=   False
      _a   :     List[str]								=   {}
      _a   :     Optional[Any]								=   False


      def _UpperCamelCase						(    self				:	str						):





            """simple docstring"""







            super().setUp()

            # fmt: off
            __snake_case      :    Union[str, Any]  			=['''[GO]''', '''[s]''', '''0''', '''1''', '''2''', '''3''', '''4''', '''5''', '''6''', '''7''', '''8''', '''9''', '''a''', '''b''', '''c''', '''d''', '''e''', '''f''', '''g''', '''h''', '''i''', '''j''', '''k''', '''l''', '''m''', '''n''', '''o''', '''p''', '''q''', '''r''', '''s''', '''t''', '''u''', '''v''', '''w''', '''x''', '''y''', '''z''']
            # fmt: on
            __snake_case      :    Optional[Any]  			=dict(zip(a							,   range(len(a						)						)						)						)

            __snake_case      :    int  			=os.path.join(self.tmpdirname							,   VOCAB_FILES_NAMES['''vocab_file''']						)
            with open(self.vocab_file							,   '''w'''							,   encoding='''utf-8'''						) as fp:
                  fp.write(json.dumps(a						) + '''\n'''						)


      def _UpperCamelCase						(    self				:	Any							,   **a				:	int						):





            """simple docstring"""







            return MgpstrTokenizer.from_pretrained(self.tmpdirname							,   **a						)


      def _UpperCamelCase						(    self				:	int							,   a				:	Any						):





            """simple docstring"""







            __snake_case      :    Dict  			='''tester'''
            __snake_case      :    str  			='''tester'''
            return input_text, output_text


      @unittest.skip('''MGP-STR always lower cases letters.'''						)
      def _UpperCamelCase						(    self				:	Any						):





            """simple docstring"""







            pass


      def _UpperCamelCase						(    self				:	Any						):





            """simple docstring"""







            __snake_case      :    List[Any]  			=self.get_tokenizers(do_lower_case=a						)
            for tokenizer in tokenizers:
                  with self.subTest(f'''{tokenizer.__class__.__name__}'''						):
                        __snake_case      :    List[Any]  			='''[SPECIAL_TOKEN]'''

                        tokenizer.add_special_tokens({'''cls_token''': special_token}						)
                        __snake_case      :    Tuple  			=tokenizer.encode([special_token]							,   add_special_tokens=a						)
                        self.assertEqual(len(a						)							,   1						)

                        __snake_case      :    int  			=tokenizer.decode(a							,   skip_special_tokens=a						)
                        self.assertTrue(special_token not in decoded						)


      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=self.get_tokenizers()
            for tokenizer in tokenizers:
                  with self.subTest(f'''{tokenizer.__class__.__name__}'''						):
                        __snake_case			,				__snake_case      :    Tuple  			=self.get_input_output_texts(a						)

                        __snake_case      :    int  			=tokenizer.tokenize(a						)
                        __snake_case      :    str  			=tokenizer.convert_tokens_to_ids(a						)
                        __snake_case      :    Union[str, Any]  			=tokenizer.encode(a							,   add_special_tokens=a						)
                        self.assertListEqual(a							,   a						)

                        __snake_case      :    Tuple  			=tokenizer.convert_ids_to_tokens(a						)
                        self.assertNotEqual(len(a						)							,   0						)
                        __snake_case      :    Optional[Any]  			=tokenizer.decode(a						)
                        self.assertIsInstance(a							,   a						)

                        self.assertEqual(text_a.replace(''' '''							,   ''''''						)							,   a						)


      @unittest.skip('''MGP-STR tokenizer only handles one sequence.'''						)
      def _UpperCamelCase						(    self				:	List[Any]						):





            """simple docstring"""







            pass







      @unittest.skip('''inputs cannot be pretokenized in MgpstrTokenizer'''						)
      def _UpperCamelCase						(    self				:	List[str]						):





            """simple docstring"""







            pass


