



import importlib
import inspect
import json
import os
import re
import shutil
import sys
from pathlib import Path
from typing import Dict, Optional, Union
from urllib import request

from huggingface_hub import HfFolder, cached_download, hf_hub_download, model_info
from packaging import version

from .. import __version__
from . import DIFFUSERS_DYNAMIC_MODULE_NAME, HF_MODULES_CACHE, logging


__lowerCamelCase     :							str  							=				(
    "https://raw.githubusercontent.com/huggingface/diffusers/{revision}/examples/community/{pipeline}.py"
)


__lowerCamelCase     :							Dict  							=				logging.get_logger(__name__)  # pylint: disable=invalid-name

def    lowerCamelCase_() ->       Dict:
						UpperCAmelCase         =     "https://pypi.org/pypi/diffusers/json"
						UpperCAmelCase         =     json.loads(request.urlopen(lowerCamelCase_    ).read()    )["releases"].keys()
						return sorted(lowerCamelCase_    ,   key=lambda lowerCamelCase_    : version.Version(lowerCamelCase_    )    )

def    lowerCamelCase_() ->       List[str]:
						# This function has already been executed if HF_MODULES_CACHE already is in the Python path.
						if HF_MODULES_CACHE in sys.path:
												return

						sys.path.append(lowerCamelCase_    )
						os.makedirs(lowerCamelCase_    ,   exist_ok=lowerCamelCase_    )
						UpperCAmelCase         =     Path(lowerCamelCase_    ) / "__init__.py"
						if not init_path.exists():
												init_path.touch()

def    lowerCamelCase_(lowerCamelCase_    ) ->       int:
						init_hf_modules()
						UpperCAmelCase         =     Path(lowerCamelCase_    ) / name
						# If the parent module does not exist yet, recursively create it.
						if not dynamic_module_path.parent.exists():
												create_dynamic_module(dynamic_module_path.parent    )
						os.makedirs(lowerCamelCase_    ,   exist_ok=lowerCamelCase_    )
						UpperCAmelCase         =     dynamic_module_path / "__init__.py"
						if not init_path.exists():
												init_path.touch()

def    lowerCamelCase_(lowerCamelCase_    ) ->       Union[str, Any]:
						with open(lowerCamelCase_    ,   "r"    ,   encoding="utf-8"    ) as f:
												UpperCAmelCase         =     f.read()

						# Imports of the form `import .xxx`
						UpperCAmelCase         =     re.findall("^\s*import\s+\.(\S+)\s*$"    ,   lowerCamelCase_    ,   flags=re.MULTILINE    )
						# Imports of the form `from .xxx import yyy`
						relative_imports += re.findall("^\s*from\s+\.(\S+)\s+import"    ,   lowerCamelCase_    ,   flags=re.MULTILINE    )
						# Unique-ify
						return list(set(lowerCamelCase_    )    )

def    lowerCamelCase_(lowerCamelCase_    ) ->       Optional[Any]:
						UpperCAmelCase         =     False
						UpperCAmelCase         =     [module_file]
						UpperCAmelCase         =     []

						# Let's recurse through all relative imports
						while not no_change:
												UpperCAmelCase         =     []
												for f in files_to_check:
																		new_imports.extend(get_relative_imports(lowerCamelCase_    )    )

												UpperCAmelCase         =     Path(lowerCamelCase_    ).parent
												UpperCAmelCase         =     [str(module_path / m    ) for m in new_imports]
												UpperCAmelCase         =     [f for f in new_import_files if f not in all_relative_imports]
												UpperCAmelCase         =     [F'{f}.py' for f in new_import_files]

												UpperCAmelCase         =     len(lowerCamelCase_    ) == 0
												all_relative_imports.extend(lowerCamelCase_    )

						return all_relative_imports

def    lowerCamelCase_(lowerCamelCase_    ) ->       Union[str, Any]:
						with open(lowerCamelCase_    ,   "r"    ,   encoding="utf-8"    ) as f:
												UpperCAmelCase         =     f.read()

						# Imports of the form `import xxx`
						UpperCAmelCase         =     re.findall("^\s*import\s+(\S+)\s*$"    ,   lowerCamelCase_    ,   flags=re.MULTILINE    )
						# Imports of the form `from xxx import yyy`
						imports += re.findall("^\s*from\s+(\S+)\s+import"    ,   lowerCamelCase_    ,   flags=re.MULTILINE    )
						# Only keep the top-level module
						UpperCAmelCase         =     [imp.split("."    )[0] for imp in imports if not imp.startswith("."    )]

						# Unique-ify and test we got them all
						UpperCAmelCase         =     list(set(lowerCamelCase_    )    )
						UpperCAmelCase         =     []
						for imp in imports:
												try:
																		importlib.import_module(lowerCamelCase_    )
												except ImportError:
																		missing_packages.append(lowerCamelCase_    )

						if len(lowerCamelCase_    ) > 0:
												raise ImportError(
												    "This modeling file requires the following packages that were not found in your environment: "
												    F'{", ".join(lowerCamelCase_    )}. Run `pip install {" ".join(lowerCamelCase_    )}`'    )

						return get_relative_imports(lowerCamelCase_    )

def    lowerCamelCase_(lowerCamelCase_    ,   lowerCamelCase_    ) ->       Union[str, Any]:
						UpperCAmelCase         =     module_path.replace(os.path.sep    ,   "."    )
						UpperCAmelCase         =     importlib.import_module(lowerCamelCase_    )

						if class_name is None:
												return find_pipeline_class(lowerCamelCase_    )
						return getattr(lowerCamelCase_    ,   lowerCamelCase_    )

def    lowerCamelCase_(lowerCamelCase_    ) ->       Union[str, Any]:
						from ..pipelines import DiffusionPipeline

						UpperCAmelCase         =     dict(inspect.getmembers(lowerCamelCase_    ,   inspect.isclass    )    )

						UpperCAmelCase         =     None
						for cls_name, cls in cls_members.items():
												if (
												    cls_name != DiffusionPipeline.__name__
												    and issubclass(cls    ,   lowerCamelCase_    )
												    and cls.__module__.split("."    )[0] != "diffusers"
												):
																		if pipeline_class is not None:
																								raise ValueError(
																								    F'Multiple classes that inherit from {DiffusionPipeline.__name__} have been found:'
																								    F' {pipeline_class.__name__}, and {cls_name}. Please make sure to define only one in'
																								    F' {loaded_module}.'    )
																		UpperCAmelCase         =     cls

						return pipeline_class

def    lowerCamelCase_(lowerCamelCase_    ,   lowerCamelCase_    ,   lowerCamelCase_ = None    ,   lowerCamelCase_ = False    ,   lowerCamelCase_ = False    ,   lowerCamelCase_ = None    ,   lowerCamelCase_ = None    ,   lowerCamelCase_ = None    ,   lowerCamelCase_ = False    ,   ) ->       int:
						UpperCAmelCase         =     str(lowerCamelCase_    )

						UpperCAmelCase         =     os.path.join(lowerCamelCase_    ,   lowerCamelCase_    )

						if os.path.isfile(lowerCamelCase_    ):
												UpperCAmelCase         =     module_file_or_url
												UpperCAmelCase         =     "local"
						elif pretrained_model_name_or_path.count("/"    ) == 0:
												UpperCAmelCase         =     get_diffusers_versions()
												# cut ".dev0"
												UpperCAmelCase         =     "v" + ".".join(__version__.split("."    )[:3]    )

												# retrieve github version that matches
												if revision is None:
																		UpperCAmelCase         =     latest_version if latest_version[1:] in available_versions else "main"
																		logger.info(F'Defaulting to latest_version: {revision}.'    )
												elif revision in available_versions:
																		UpperCAmelCase         =     F'v{revision}'
												elif revision == "main":
																		UpperCAmelCase         =     revision
												else:
																		raise ValueError(
																		    F'`custom_revision`: {revision} does not exist. Please make sure to choose one of'
																		    F' {", ".join(available_versions + ["main"]    )}.'    )

												# community pipeline on GitHub
												UpperCAmelCase         =     COMMUNITY_PIPELINES_URL.format(revision=lowerCamelCase_    ,   pipeline=lowerCamelCase_    )
												try:
																		UpperCAmelCase         =     cached_download(
																		    lowerCamelCase_    ,   cache_dir=lowerCamelCase_    ,   force_download=lowerCamelCase_    ,   proxies=lowerCamelCase_    ,   resume_download=lowerCamelCase_    ,   local_files_only=lowerCamelCase_    ,   use_auth_token=lowerCamelCase_    ,   )
																		UpperCAmelCase         =     "git"
																		UpperCAmelCase         =     pretrained_model_name_or_path + ".py"
												except EnvironmentError:
																		logger.error(F'Could not locate the {module_file} inside {pretrained_model_name_or_path}.'    )
																		raise
						else:
												try:
																		# Load from URL or cache if already cached
																		UpperCAmelCase         =     hf_hub_download(
																		    lowerCamelCase_    ,   lowerCamelCase_    ,   cache_dir=lowerCamelCase_    ,   force_download=lowerCamelCase_    ,   proxies=lowerCamelCase_    ,   resume_download=lowerCamelCase_    ,   local_files_only=lowerCamelCase_    ,   use_auth_token=lowerCamelCase_    ,   )
																		UpperCAmelCase         =     os.path.join("local"    ,   "--".join(pretrained_model_name_or_path.split("/"    )    )    )
												except EnvironmentError:
																		logger.error(F'Could not locate the {module_file} inside {pretrained_model_name_or_path}.'    )
																		raise

    # Check we have all the requirements in our environment
						UpperCAmelCase         =     check_imports(lowerCamelCase_    )

						# Now we move the module inside our cached dynamic modules.
						UpperCAmelCase         =     DIFFUSERS_DYNAMIC_MODULE_NAME + os.path.sep + submodule
						create_dynamic_module(lowerCamelCase_    )
						UpperCAmelCase         =     Path(lowerCamelCase_    ) / full_submodule
						if submodule == "local" or submodule == "git":
												# We always copy local files (we could hash the file to see if there was a change, and give them the name of
												# that hash, to only copy when there is a modification but it seems overkill for now).
												# The only reason we do the copy is to avoid putting too many folders in sys.path.
												shutil.copy(lowerCamelCase_    ,   submodule_path / module_file    )
												for module_needed in modules_needed:
																		UpperCAmelCase         =     F'{module_needed}.py'
																		shutil.copy(os.path.join(lowerCamelCase_    ,   lowerCamelCase_    )    ,   submodule_path / module_needed    )
						else:
												# Get the commit hash
												# TODO: we will get this info in the etag soon, so retrieve it from there and not here.
												if isinstance(lowerCamelCase_    ,   lowerCamelCase_    ):
																		UpperCAmelCase         =     use_auth_token
												elif use_auth_token is True:
																		UpperCAmelCase         =     HfFolder.get_token()
												else:
																		UpperCAmelCase         =     None

												UpperCAmelCase         =     model_info(lowerCamelCase_    ,   revision=lowerCamelCase_    ,   token=lowerCamelCase_    ).sha

												# The module file will end up being placed in a subfolder with the git hash of the repo. This way we get the
												# benefit of versioning.
												UpperCAmelCase         =     submodule_path / commit_hash
												UpperCAmelCase         =     full_submodule + os.path.sep + commit_hash
												create_dynamic_module(lowerCamelCase_    )

												if not (submodule_path / module_file).exists():
																		shutil.copy(lowerCamelCase_    ,   submodule_path / module_file    )
												# Make sure we also have every file with relative
												for module_needed in modules_needed:
																		if not (submodule_path / module_needed).exists():
																								get_cached_module_file(
																								    lowerCamelCase_    ,   F'{module_needed}.py'    ,   cache_dir=lowerCamelCase_    ,   force_download=lowerCamelCase_    ,   resume_download=lowerCamelCase_    ,   proxies=lowerCamelCase_    ,   use_auth_token=lowerCamelCase_    ,   revision=lowerCamelCase_    ,   local_files_only=lowerCamelCase_    ,   )
						return os.path.join(lowerCamelCase_    ,   lowerCamelCase_    )





def    lowerCamelCase_(lowerCamelCase_    ,   lowerCamelCase_    ,   lowerCamelCase_ = None    ,   lowerCamelCase_ = None    ,   lowerCamelCase_ = False    ,   lowerCamelCase_ = False    ,   lowerCamelCase_ = None    ,   lowerCamelCase_ = None    ,   lowerCamelCase_ = None    ,   lowerCamelCase_ = False    ,   **lowerCamelCase_    ,   ) ->       Dict:
						UpperCAmelCase         =     get_cached_module_file(
						    lowerCamelCase_    ,   lowerCamelCase_    ,   cache_dir=lowerCamelCase_    ,   force_download=lowerCamelCase_    ,   resume_download=lowerCamelCase_    ,   proxies=lowerCamelCase_    ,   use_auth_token=lowerCamelCase_    ,   revision=lowerCamelCase_    ,   local_files_only=lowerCamelCase_    ,   )
						return get_class_in_module(lowerCamelCase_    ,   final_module.replace(".py"    ,   ""    )    )








import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase     :							Any  							=				get_tests_dir("fixtures/test_sentencepiece.model")





@require_sentencepiece
@require_tokenizers
class  __magic_name__   (							A__,  unittest.TestCase						):
		lowercase      :       Any          =ReformerTokenizer
		lowercase      :       str          =ReformerTokenizerFast
		lowercase      :       str          =True
		lowercase      :       Optional[int]          =False
		lowercase      :       List[str]          =True






		def SCREAMING_SNAKE_CASE_			(						self      :     Union[str, Any]      )				->					Any:
								'''simple docstring'''







								super().setUp()

								UpperCAmelCase         =     ReformerTokenizer(UpperCamelCase__				,		keep_accents=UpperCamelCase__      )
								tokenizer.save_pretrained(self.tmpdirname      )






		def SCREAMING_SNAKE_CASE_			(						self      :     List[Any]      )				->					Optional[int]:
								'''simple docstring'''







								UpperCAmelCase         =     "<s>"
								UpperCAmelCase         =     1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__      )				,		UpperCamelCase__      )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__      )				,		UpperCamelCase__      )






		def SCREAMING_SNAKE_CASE_			(						self      :     int      )				->					str:
								'''simple docstring'''







								UpperCAmelCase         =     list(self.get_tokenizer().get_vocab().keys()      )

								self.assertEqual(vocab_keys[0]				,		"<unk>"      )
								self.assertEqual(vocab_keys[1]				,		"<s>"      )
								self.assertEqual(vocab_keys[-1]				,		"j"      )
								self.assertEqual(len(UpperCamelCase__      )				,		10_00      )






		def SCREAMING_SNAKE_CASE_			(						self      :     Optional[Any]      )				->					Dict:
								'''simple docstring'''







								self.assertEqual(self.get_tokenizer().vocab_size				,		10_00      )






		def SCREAMING_SNAKE_CASE_			(						self      :     Union[str, Any]      )				->					Tuple:
								'''simple docstring'''







								if not self.test_rust_tokenizer:
														return

								UpperCAmelCase         =     self.get_tokenizer()
								UpperCAmelCase         =     self.get_rust_tokenizer()

								UpperCAmelCase         =     "I was born in 92000, and this is falsé."

								UpperCAmelCase         =     tokenizer.tokenize(UpperCamelCase__      )
								UpperCAmelCase         =     rust_tokenizer.tokenize(UpperCamelCase__      )
								self.assertListEqual(UpperCamelCase__				,		UpperCamelCase__      )

								UpperCAmelCase         =     tokenizer.encode(UpperCamelCase__				,		add_special_tokens=UpperCamelCase__      )
								UpperCAmelCase         =     rust_tokenizer.encode(UpperCamelCase__				,		add_special_tokens=UpperCamelCase__      )
								self.assertListEqual(UpperCamelCase__				,		UpperCamelCase__      )

								UpperCAmelCase         =     self.get_rust_tokenizer()
								UpperCAmelCase         =     tokenizer.encode(UpperCamelCase__      )
								UpperCAmelCase         =     rust_tokenizer.encode(UpperCamelCase__      )
								self.assertListEqual(UpperCamelCase__				,		UpperCamelCase__      )






		def SCREAMING_SNAKE_CASE_			(						self      :     int				,		UpperCamelCase__      :     Union[str, Any]=15      )				->					List[str]:
								'''simple docstring'''







								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																				UpperCAmelCase         =     self.rust_tokenizer_class.from_pretrained(UpperCamelCase__				,		**UpperCamelCase__      )

																				# Simple input
																				UpperCAmelCase         =     "This is a simple input"
																				UpperCAmelCase         =     ["This is a simple input 1", "This is a simple input 2"]
																				UpperCAmelCase         =     ("This is a simple input", "This is a pair")
																				UpperCAmelCase         =     [
																				    ("This is a simple input 1", "This is a simple input 2"),
																				    ("This is a simple pair 1", "This is a simple pair 2"),
																				]

																				# Simple input tests
																				self.assertRaises(UpperCamelCase__				,		tokenizer_r.encode				,		UpperCamelCase__				,		max_length=UpperCamelCase__				,		padding="max_length"      )

																				# Simple input
																				self.assertRaises(UpperCamelCase__				,		tokenizer_r.encode_plus				,		UpperCamelCase__				,		max_length=UpperCamelCase__				,		padding="max_length"      )

																				# Simple input
																				self.assertRaises(
																				    UpperCamelCase__				,		tokenizer_r.batch_encode_plus				,		UpperCamelCase__				,		max_length=UpperCamelCase__				,		padding="max_length"				,		)

																				# Pair input
																				self.assertRaises(UpperCamelCase__				,		tokenizer_r.encode				,		UpperCamelCase__				,		max_length=UpperCamelCase__				,		padding="max_length"      )

																				# Pair input
																				self.assertRaises(UpperCamelCase__				,		tokenizer_r.encode_plus				,		UpperCamelCase__				,		max_length=UpperCamelCase__				,		padding="max_length"      )

																				# Pair input
																				self.assertRaises(
																				    UpperCamelCase__				,		tokenizer_r.batch_encode_plus				,		UpperCamelCase__				,		max_length=UpperCamelCase__				,		padding="max_length"				,		)






		def SCREAMING_SNAKE_CASE_			(						self      :     List[str]      )				->					Any:
								'''simple docstring'''







								pass






		def SCREAMING_SNAKE_CASE_			(						self      :     Union[str, Any]      )				->					Optional[int]:
								'''simple docstring'''







								UpperCAmelCase         =     ReformerTokenizer(UpperCamelCase__				,		keep_accents=UpperCamelCase__      )

								UpperCAmelCase         =     tokenizer.tokenize("This is a test"      )
								self.assertListEqual(UpperCamelCase__				,		["▁This", "▁is", "▁a", "▁t", "est"]      )

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(UpperCamelCase__      )				,		[2_85, 46, 10, 1_70, 3_82]				,		)

								UpperCAmelCase         =     tokenizer.tokenize("I was born in 92000, and this is falsé."      )
								self.assertListEqual(
								    UpperCamelCase__				,		[
								        SPIECE_UNDERLINE + "I",
								        SPIECE_UNDERLINE + "was",
								        SPIECE_UNDERLINE + "b",
								        "or",
								        "n",
								        SPIECE_UNDERLINE + "in",
								        SPIECE_UNDERLINE + "",
								        "9",
								        "2",
								        "0",
								        "0",
								        "0",
								        ",",
								        SPIECE_UNDERLINE + "and",
								        SPIECE_UNDERLINE + "this",
								        SPIECE_UNDERLINE + "is",
								        SPIECE_UNDERLINE + "f",
								        "al",
								        "s",
								        "é",
								        ".",
								    ]				,		)
								UpperCAmelCase         =     tokenizer.convert_tokens_to_ids(UpperCamelCase__      )
								self.assertListEqual(
								    UpperCamelCase__				,		[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]				,		)

								UpperCAmelCase         =     tokenizer.convert_ids_to_tokens(UpperCamelCase__      )
								self.assertListEqual(
								    UpperCamelCase__				,		[
								        SPIECE_UNDERLINE + "I",
								        SPIECE_UNDERLINE + "was",
								        SPIECE_UNDERLINE + "b",
								        "or",
								        "n",
								        SPIECE_UNDERLINE + "in",
								        SPIECE_UNDERLINE + "",
								        "<unk>",
								        "2",
								        "0",
								        "0",
								        "0",
								        ",",
								        SPIECE_UNDERLINE + "and",
								        SPIECE_UNDERLINE + "this",
								        SPIECE_UNDERLINE + "is",
								        SPIECE_UNDERLINE + "f",
								        "al",
								        "s",
								        "<unk>",
								        ".",
								    ]				,		)






		@cached_property
		def SCREAMING_SNAKE_CASE_			(						self      :     Optional[Any]      )				->					Union[str, Any]:
								'''simple docstring'''







								return ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment"      )






		@slow
		def SCREAMING_SNAKE_CASE_			(						self      :     List[str]      )				->					str:
								'''simple docstring'''







								UpperCAmelCase         =     "Hello World!"
								UpperCAmelCase         =     [1_26, 32, 2_62, 1_52, 38, 72, 2_87]

								self.assertListEqual(UpperCamelCase__				,		self.big_tokenizer.encode(UpperCamelCase__      )      )






		@slow
		def SCREAMING_SNAKE_CASE_			(						self      :     Optional[int]      )				->					Any:
								'''simple docstring'''







								UpperCAmelCase         =     (
								    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
								    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
								)
								UpperCAmelCase         =     [
								    1_08,
								    2_65,
								    24,
								    1_11,
								    4,
								    2_58,
								    1_56,
								    35,
								    28,
								    2_75,
								    3,
								    2_59,
								    2_97,
								    2_60,
								    84,
								    4,
								    35,
								    1_10,
								    44,
								    8,
								    2_59,
								    91,
								    2_68,
								    21,
								    11,
								    2_09,
								    2_74,
								    1_09,
								    2_66,
								    2_77,
								    1_17,
								    86,
								    93,
								    3_15,
								    2_58,
								    2_78,
								    2_58,
								    2_77,
								    2_58,
								    0,
								    2_58,
								    2_88,
								    2_58,
								    3_19,
								    2_58,
								    0,
								    2_58,
								    0,
								    2_58,
								    0,
								    2_58,
								    0,
								    2_58,
								    2_87,
								    2_58,
								    3_15,
								    2_58,
								    2_89,
								    2_58,
								    2_78,
								    99,
								    2_69,
								    2_66,
								    2_62,
								    8,
								    2_59,
								    2_41,
								    4,
								    2_17,
								    2_30,
								    2_68,
								    2_66,
								    55,
								    1_68,
								    1_06,
								    75,
								    1_93,
								    2_66,
								    2_23,
								    27,
								    49,
								    26,
								    2_82,
								    25,
								    2_64,
								    2_99,
								    19,
								    26,
								    0,
								    2_58,
								    2_77,
								    1_17,
								    86,
								    93,
								    1_76,
								    1_83,
								    2_70,
								    11,
								    2_62,
								    42,
								    61,
								    2_65,
								]

								self.assertListEqual(UpperCamelCase__				,		self.big_tokenizer.encode(UpperCamelCase__      )      )






		@require_torch
		@slow
		def SCREAMING_SNAKE_CASE_			(						self      :     Optional[Any]      )				->					List[str]:
								'''simple docstring'''







								import torch

								from transformers import ReformerConfig, ReformerModel

								# Build sequence
								UpperCAmelCase         =     list(self.big_tokenizer.get_vocab().keys()      )[:10]
								UpperCAmelCase         =     " ".join(UpperCamelCase__      )
								UpperCAmelCase         =     self.big_tokenizer.encode_plus(UpperCamelCase__				,		return_tensors="pt"      )
								UpperCAmelCase         =     self.big_tokenizer.batch_encode_plus([sequence, sequence]				,		return_tensors="pt"      )

								UpperCAmelCase         =     ReformerConfig()
								# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
								UpperCAmelCase         =     encoded_sequence["input_ids"].shape
								UpperCAmelCase         =     ReformerModel(UpperCamelCase__      )

								# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
								assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

								with torch.no_grad():
														model(**UpperCamelCase__      )
														model(**UpperCamelCase__      )







		@slow
		def SCREAMING_SNAKE_CASE_			(						self      :     str      )				->					List[str]:
								'''simple docstring'''







								UpperCAmelCase         =     {"input_ids": [[1_08, 2_65, 24, 1_11, 4, 2_58, 1_56, 7, 51, 2_79, 58, 7, 76, 25, 69, 2_78], [1_40, 2_43, 2_64, 1_34, 17, 2_67, 77, 2_63, 22, 2_62, 2_97, 2_58, 3_04, 1_77, 2_79, 2_66, 14, 89, 13, 35, 2_61, 2_99, 2_72, 1_37, 2_75, 2_78]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
								# fmt: on

								# This tokenizer does not know some characters like ")".
								# That is the reason why we use very simple texts here.
								# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
								UpperCAmelCase         =     [
								    "This is a very simple sentence.",
								    "The quick brown fox jumps over the lazy dog.",
								]

								self.tokenizer_integration_test_util(
								    expected_encoding=UpperCamelCase__				,		model_name="google/reformer-crime-and-punishment"				,		revision="0e6c3decb8211d49bf881013425dc8b0448b3f5a"				,		padding=UpperCamelCase__				,		sequences=UpperCamelCase__				,		)



