
'''simple docstring'''



import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__      :		List[str]       							=  get_tests_dir("fixtures/test_sentencepiece.model")

@require_sentencepiece
@require_tokenizers
class    _a				(UpperCamelCase_  ,					unittest.TestCase):





       """simple docstring"""




       SCREAMING_SNAKE_CASE      =  ReformerTokenizer
       SCREAMING_SNAKE_CASE      =  ReformerTokenizerFast
       SCREAMING_SNAKE_CASE      =  True
       SCREAMING_SNAKE_CASE      =  False
       SCREAMING_SNAKE_CASE      =  True






       def        UpperCamelCase				(							self      )    ->			List[str]:
          super().setUp()

          _SCREAMING_SNAKE_CASE      =				ReformerTokenizer(_a       ,							keep_accents=_a      )
          tokenizer.save_pretrained(self.tmpdirname      )






       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				"""<s>"""
          _SCREAMING_SNAKE_CASE      =				1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a      )       ,							_a      )
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a      )       ,							_a      )






       def        UpperCamelCase				(							self      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				list(self.get_tokenizer().get_vocab().keys()      )

          self.assertEqual(vocab_keys[0]       ,							"""<unk>"""      )
          self.assertEqual(vocab_keys[1]       ,							"""<s>"""      )
          self.assertEqual(vocab_keys[-1]       ,							"""j"""      )
          self.assertEqual(len(_a      )       ,							10_00      )






       def        UpperCamelCase				(							self      )    ->			Any:
          self.assertEqual(self.get_tokenizer().vocab_size       ,							10_00      )






       def        UpperCamelCase				(							self      )    ->			Optional[int]:


          if not self.test_rust_tokenizer:
             return

          _SCREAMING_SNAKE_CASE      =				self.get_tokenizer()
          _SCREAMING_SNAKE_CASE      =				self.get_rust_tokenizer()

          _SCREAMING_SNAKE_CASE      =				"""I was born in 92000, and this is falsé."""

          _SCREAMING_SNAKE_CASE      =				tokenizer.tokenize(_a      )
          _SCREAMING_SNAKE_CASE      =				rust_tokenizer.tokenize(_a      )
          self.assertListEqual(_a       ,							_a      )

          _SCREAMING_SNAKE_CASE      =				tokenizer.encode(_a       ,							add_special_tokens=_a      )
          _SCREAMING_SNAKE_CASE      =				rust_tokenizer.encode(_a       ,							add_special_tokens=_a      )
          self.assertListEqual(_a       ,							_a      )

          _SCREAMING_SNAKE_CASE      =				self.get_rust_tokenizer()
          _SCREAMING_SNAKE_CASE      =				tokenizer.encode(_a      )
          _SCREAMING_SNAKE_CASE      =				rust_tokenizer.encode(_a      )
          self.assertListEqual(_a       ,							_a      )






       def        UpperCamelCase				(							self       ,							A__=15      )    ->			Union[str, Any]:


          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
                _SCREAMING_SNAKE_CASE      =				self.rust_tokenizer_class.from_pretrained(_a       ,							**_a      )

                # Simple input
                _SCREAMING_SNAKE_CASE      =				"""This is a simple input"""
                _SCREAMING_SNAKE_CASE      =				["""This is a simple input 1""", """This is a simple input 2"""]
                _SCREAMING_SNAKE_CASE      =				("""This is a simple input""", """This is a pair""")
                _SCREAMING_SNAKE_CASE      =				[
                    ("""This is a simple input 1""", """This is a simple input 2"""),
                    ("""This is a simple pair 1""", """This is a simple pair 2"""),
                ]

                # Simple input tests
                self.assertRaises(_a       ,							tokenizer_r.encode       ,							_a       ,							max_length=_a       ,							padding="""max_length"""      )

                # Simple input
                self.assertRaises(_a       ,							tokenizer_r.encode_plus       ,							_a       ,							max_length=_a       ,							padding="""max_length"""      )

                # Simple input
                self.assertRaises(
                    _a       ,							tokenizer_r.batch_encode_plus       ,							_a       ,							max_length=_a       ,							padding="""max_length"""       ,							)

                # Pair input
                self.assertRaises(_a       ,							tokenizer_r.encode       ,							_a       ,							max_length=_a       ,							padding="""max_length"""      )

                # Pair input
                self.assertRaises(_a       ,							tokenizer_r.encode_plus       ,							_a       ,							max_length=_a       ,							padding="""max_length"""      )

                # Pair input
                self.assertRaises(
                    _a       ,							tokenizer_r.batch_encode_plus       ,							_a       ,							max_length=_a       ,							padding="""max_length"""       ,							)






       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          pass






       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				ReformerTokenizer(_a       ,							keep_accents=_a      )

          _SCREAMING_SNAKE_CASE      =				tokenizer.tokenize("""This is a test"""      )
          self.assertListEqual(_a       ,							["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]      )

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(_a      )       ,							[2_85, 46, 10, 1_70, 3_82]       ,							)

          _SCREAMING_SNAKE_CASE      =				tokenizer.tokenize("""I was born in 92000, and this is falsé."""      )
          self.assertListEqual(
              _a       ,							[
                  SPIECE_UNDERLINE + """I""",
                  SPIECE_UNDERLINE + """was""",
                  SPIECE_UNDERLINE + """b""",
                  """or""",
                  """n""",
                  SPIECE_UNDERLINE + """in""",
                  SPIECE_UNDERLINE + """""",
                  """9""",
                  """2""",
                  """0""",
                  """0""",
                  """0""",
                  """,""",
                  SPIECE_UNDERLINE + """and""",
                  SPIECE_UNDERLINE + """this""",
                  SPIECE_UNDERLINE + """is""",
                  SPIECE_UNDERLINE + """f""",
                  """al""",
                  """s""",
                  """é""",
                  """.""",
              ]       ,							)
          _SCREAMING_SNAKE_CASE      =				tokenizer.convert_tokens_to_ids(_a      )
          self.assertListEqual(
              _a       ,							[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]       ,							)

          _SCREAMING_SNAKE_CASE      =				tokenizer.convert_ids_to_tokens(_a      )
          self.assertListEqual(
              _a       ,							[
                  SPIECE_UNDERLINE + """I""",
                  SPIECE_UNDERLINE + """was""",
                  SPIECE_UNDERLINE + """b""",
                  """or""",
                  """n""",
                  SPIECE_UNDERLINE + """in""",
                  SPIECE_UNDERLINE + """""",
                  """<unk>""",
                  """2""",
                  """0""",
                  """0""",
                  """0""",
                  """,""",
                  SPIECE_UNDERLINE + """and""",
                  SPIECE_UNDERLINE + """this""",
                  SPIECE_UNDERLINE + """is""",
                  SPIECE_UNDERLINE + """f""",
                  """al""",
                  """s""",
                  """<unk>""",
                  """.""",
              ]       ,							)






       @cached_property
       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          return ReformerTokenizer.from_pretrained("""google/reformer-crime-and-punishment"""      )






       @slow
       def        UpperCamelCase				(							self      )    ->			Tuple:
          _SCREAMING_SNAKE_CASE      =				"""Hello World!"""
          _SCREAMING_SNAKE_CASE      =				[1_26, 32, 2_62, 1_52, 38, 72, 2_87]

          self.assertListEqual(_a       ,							self.big_tokenizer.encode(_a      )      )






       @slow
       def        UpperCamelCase				(							self      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE      =				(
              """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
              """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
          )
          _SCREAMING_SNAKE_CASE      =				[
              1_08,
              2_65,
              24,
              1_11,
              4,
              2_58,
              1_56,
              35,
              28,
              2_75,
              3,
              2_59,
              2_97,
              2_60,
              84,
              4,
              35,
              1_10,
              44,
              8,
              2_59,
              91,
              2_68,
              21,
              11,
              2_09,
              2_74,
              1_09,
              2_66,
              2_77,
              1_17,
              86,
              93,
              3_15,
              2_58,
              2_78,
              2_58,
              2_77,
              2_58,
              0,
              2_58,
              2_88,
              2_58,
              3_19,
              2_58,
              0,
              2_58,
              0,
              2_58,
              0,
              2_58,
              0,
              2_58,
              2_87,
              2_58,
              3_15,
              2_58,
              2_89,
              2_58,
              2_78,
              99,
              2_69,
              2_66,
              2_62,
              8,
              2_59,
              2_41,
              4,
              2_17,
              2_30,
              2_68,
              2_66,
              55,
              1_68,
              1_06,
              75,
              1_93,
              2_66,
              2_23,
              27,
              49,
              26,
              2_82,
              25,
              2_64,
              2_99,
              19,
              26,
              0,
              2_58,
              2_77,
              1_17,
              86,
              93,
              1_76,
              1_83,
              2_70,
              11,
              2_62,
              42,
              61,
              2_65,
          ]

          self.assertListEqual(_a       ,							self.big_tokenizer.encode(_a      )      )






       @require_torch
       @slow
       def        UpperCamelCase				(							self      )    ->			Tuple:
          import torch

          from transformers import ReformerConfig, ReformerModel

          # Build sequence
          _SCREAMING_SNAKE_CASE      =				list(self.big_tokenizer.get_vocab().keys()      )[:10]
          _SCREAMING_SNAKE_CASE      =				""" """.join(_a      )
          _SCREAMING_SNAKE_CASE      =				self.big_tokenizer.encode_plus(_a       ,							return_tensors="""pt"""      )
          _SCREAMING_SNAKE_CASE      =				self.big_tokenizer.batch_encode_plus([sequence, sequence]       ,							return_tensors="""pt"""      )

          _SCREAMING_SNAKE_CASE      =				ReformerConfig()
          # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
          _SCREAMING_SNAKE_CASE      =				encoded_sequence["""input_ids"""].shape
          _SCREAMING_SNAKE_CASE      =				ReformerModel(_a      )

          # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
          assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

          with torch.no_grad():
             model(**_a      )
             model(**_a      )






       @slow
       def        UpperCamelCase				(							self      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				{"""input_ids""": [[1_08, 2_65, 24, 1_11, 4, 2_58, 1_56, 7, 51, 2_79, 58, 7, 76, 25, 69, 2_78], [1_40, 2_43, 2_64, 1_34, 17, 2_67, 77, 2_63, 22, 2_62, 2_97, 2_58, 3_04, 1_77, 2_79, 2_66, 14, 89, 13, 35, 2_61, 2_99, 2_72, 1_37, 2_75, 2_78]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
          # fmt: on

          # This tokenizer does not know some characters like ")".
          # That is the reason why we use very simple texts here.
          # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
          _SCREAMING_SNAKE_CASE      =				[
              """This is a very simple sentence.""",
              """The quick brown fox jumps over the lazy dog.""",
          ]

          self.tokenizer_integration_test_util(
              expected_encoding=_a       ,							model_name="""google/reformer-crime-and-punishment"""       ,							revision="""0e6c3decb8211d49bf881013425dc8b0448b3f5a"""       ,							padding=_a       ,							sequences=_a       ,							)



'''simple docstring'''



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__      :		List[Any]       							=  logging.get_logger(__name__)


UpperCamelCase__      :		Any       							=  "▁"

UpperCamelCase__      :		Any       							=  {"vocab_file": "spiece.model"}

UpperCamelCase__      :		int       							=  {
    "vocab_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"
        )
    }
}

UpperCamelCase__      :		Optional[int]       							=  {
    "google/reformer-crime-and-punishment": 524_288,
}

class    _a				(_lowerCamelCase):





       """simple docstring"""




       SCREAMING_SNAKE_CASE      =  VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE      =  PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE      =  ['input_ids', 'attention_mask']






       def __init__(							self       ,							A__       ,							A__="</s>"       ,							A__="<unk>"       ,							A__=[]       ,							A__ = None       ,							**A__       ,							)    ->			None:
          _SCREAMING_SNAKE_CASE      =				{} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              eos_token=A__       ,							unk_token=A__       ,							additional_special_tokens=A__       ,							sp_model_kwargs=self.sp_model_kwargs       ,							**A__       ,							)

          _SCREAMING_SNAKE_CASE      =				vocab_file
          _SCREAMING_SNAKE_CASE      =				spm.SentencePieceProcessor(**self.sp_model_kwargs      )
          self.sp_model.Load(A__      )






       @property
       def        UpperCamelCase				(							self      )    ->			Any:
          return self.sp_model.get_piece_size()






       def        UpperCamelCase				(							self      )    ->			Dict[str, int]:
          _SCREAMING_SNAKE_CASE      =				{self.convert_ids_to_tokens(A__      ): i for i in range(self.vocab_size      )}
          vocab.update(self.added_tokens_encoder      )
          return vocab






       def __getstate__(							self      )    ->			int:
          _SCREAMING_SNAKE_CASE      =				self.__dict__.copy()
          _SCREAMING_SNAKE_CASE      =				None
          return state






       def __setstate__(							self       ,							A__      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				d

          # for backward compatibility
          if not hasattr(self       ,							"""sp_model_kwargs"""      ):
             _SCREAMING_SNAKE_CASE      =				{}

          _SCREAMING_SNAKE_CASE      =				spm.SentencePieceProcessor(**self.sp_model_kwargs      )
          self.sp_model.Load(self.vocab_file      )






       def        UpperCamelCase				(							self       ,							A__      )    ->			List[str]:
          return self.sp_model.encode(A__       ,							out_type=A__      )






       def        UpperCamelCase				(							self       ,							A__      )    ->			Union[str, Any]:
          return self.sp_model.piece_to_id(A__      )






       def        UpperCamelCase				(							self       ,							A__      )    ->			List[Any]:
          if index < self.sp_model.get_piece_size():
             _SCREAMING_SNAKE_CASE      =				self.sp_model.IdToPiece(A__      )
          return token






       def        UpperCamelCase				(							self       ,							A__      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				[]
          _SCREAMING_SNAKE_CASE      =				""""""
          for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                out_string += self.sp_model.decode(A__      ) + token
                _SCREAMING_SNAKE_CASE      =				[]
             else:
                current_sub_tokens.append(A__      )
          out_string += self.sp_model.decode(A__      )
          return out_string.strip()






       def        UpperCamelCase				(							self       ,							A__       ,							A__ = None      )    ->			Tuple[str]:
          if not os.path.isdir(A__      ):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
             return
          _SCREAMING_SNAKE_CASE      =				os.path.join(
              A__       ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

          if os.path.abspath(self.vocab_file      ) != os.path.abspath(A__      ) and os.path.isfile(self.vocab_file      ):
             copyfile(self.vocab_file       ,							A__      )
          elif not os.path.isfile(self.vocab_file      ):
             with open(A__       ,							"""wb"""      ) as fi:
                _SCREAMING_SNAKE_CASE      =				self.sp_model.serialized_model_proto()
                fi.write(A__      )

          return (out_vocab_file,)






