



SCREAMING_SNAKE_CASE__       =      '''0.18.2'''

from .configuration_utils import ConfigMixin
from .utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_onnx_available,
    is_scipy_available,
    is_torch_available,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    logging,
)


try:
       if not is_onnx_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_onnx_objects import *  # noqa F403
else:
       from .pipelines import OnnxRuntimeModel

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_pt_objects import *  # noqa F403
else:
       from .models import (
           AutoencoderKL,
           ControlNetModel,
           ModelMixin,
           PriorTransformer,
           TaFilmDecoder,
           TransformeraDModel,
           UNetaDModel,
           UNetaDConditionModel,
           UNetaDModel,
           UNetaDConditionModel,
           VQModel,
       )
       from .optimization import (
           get_constant_schedule,
           get_constant_schedule_with_warmup,
           get_cosine_schedule_with_warmup,
           get_cosine_with_hard_restarts_schedule_with_warmup,
           get_linear_schedule_with_warmup,
           get_polynomial_decay_schedule_with_warmup,
           get_scheduler,
       )
       from .pipelines import (
           AudioPipelineOutput,
           ConsistencyModelPipeline,
           DanceDiffusionPipeline,
           DDIMPipeline,
           DDPMPipeline,
           DiffusionPipeline,
           DiTPipeline,
           ImagePipelineOutput,
           KarrasVePipeline,
           LDMPipeline,
           LDMSuperResolutionPipeline,
           PNDMPipeline,
           RePaintPipeline,
           ScoreSdeVePipeline,
       )
       from .schedulers import (
           CMStochasticIterativeScheduler,
           DDIMInverseScheduler,
           DDIMParallelScheduler,
           DDIMScheduler,
           DDPMParallelScheduler,
           DDPMScheduler,
           DEISMultistepScheduler,
           DPMSolverMultistepInverseScheduler,
           DPMSolverMultistepScheduler,
           DPMSolverSinglestepScheduler,
           EulerAncestralDiscreteScheduler,
           EulerDiscreteScheduler,
           HeunDiscreteScheduler,
           IPNDMScheduler,
           KarrasVeScheduler,
           KDPMaAncestralDiscreteScheduler,
           KDPMaDiscreteScheduler,
           PNDMScheduler,
           RePaintScheduler,
           SchedulerMixin,
           ScoreSdeVeScheduler,
           UnCLIPScheduler,
           UniPCMultistepScheduler,
           VQDiffusionScheduler,
       )
       from .training_utils import EMAModel

try:
       if not (is_torch_available() and is_scipy_available()):
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_torch_and_scipy_objects import *  # noqa F403
else:
       from .schedulers import LMSDiscreteScheduler

try:
       if not (is_torch_available() and is_torchsde_available()):
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_torch_and_torchsde_objects import *  # noqa F403
else:
       from .schedulers import DPMSolverSDEScheduler

try:
       if not (is_torch_available() and is_transformers_available()):
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
       from .pipelines import (
           AltDiffusionImgaImgPipeline,
           AltDiffusionPipeline,
           AudioLDMPipeline,
           CycleDiffusionPipeline,
           IFImgaImgPipeline,
           IFImgaImgSuperResolutionPipeline,
           IFInpaintingPipeline,
           IFInpaintingSuperResolutionPipeline,
           IFPipeline,
           IFSuperResolutionPipeline,
           ImageTextPipelineOutput,
           KandinskyImgaImgPipeline,
           KandinskyInpaintPipeline,
           KandinskyPipeline,
           KandinskyPriorPipeline,
           KandinskyVaaControlnetImgaImgPipeline,
           KandinskyVaaControlnetPipeline,
           KandinskyVaaImgaImgPipeline,
           KandinskyVaaInpaintPipeline,
           KandinskyVaaPipeline,
           KandinskyVaaPriorEmbaEmbPipeline,
           KandinskyVaaPriorPipeline,
           LDMTextToImagePipeline,
           PaintByExamplePipeline,
           SemanticStableDiffusionPipeline,
           ShapEImgaImgPipeline,
           ShapEPipeline,
           StableDiffusionAttendAndExcitePipeline,
           StableDiffusionControlNetImgaImgPipeline,
           StableDiffusionControlNetInpaintPipeline,
           StableDiffusionControlNetPipeline,
           StableDiffusionDepthaImgPipeline,
           StableDiffusionDiffEditPipeline,
           StableDiffusionImageVariationPipeline,
           StableDiffusionImgaImgPipeline,
           StableDiffusionInpaintPipeline,
           StableDiffusionInpaintPipelineLegacy,
           StableDiffusionInstructPixaPixPipeline,
           StableDiffusionLatentUpscalePipeline,
           StableDiffusionLDMaDPipeline,
           StableDiffusionModelEditingPipeline,
           StableDiffusionPanoramaPipeline,
           StableDiffusionParadigmsPipeline,
           StableDiffusionPipeline,
           StableDiffusionPipelineSafe,
           StableDiffusionPixaPixZeroPipeline,
           StableDiffusionSAGPipeline,
           StableDiffusionUpscalePipeline,
           StableUnCLIPImgaImgPipeline,
           StableUnCLIPPipeline,
           TextToVideoSDPipeline,
           TextToVideoZeroPipeline,
           UnCLIPImageVariationPipeline,
           UnCLIPPipeline,
           UniDiffuserModel,
           UniDiffuserPipeline,
           UniDiffuserTextDecoder,
           VersatileDiffusionDualGuidedPipeline,
           VersatileDiffusionImageVariationPipeline,
           VersatileDiffusionPipeline,
           VersatileDiffusionTextToImagePipeline,
           VideoToVideoSDPipeline,
           VQDiffusionPipeline,
       )

try:
       if not (is_torch_available() and is_transformers_available() and is_invisible_watermark_available()):
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_torch_and_transformers_and_invisible_watermark_objects import *  # noqa F403
else:
       from .pipelines import StableDiffusionXLImgaImgPipeline, StableDiffusionXLPipeline

try:
       if not (is_torch_available() and is_transformers_available() and is_k_diffusion_available()):
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_torch_and_transformers_and_k_diffusion_objects import *  # noqa F403
else:
       from .pipelines import StableDiffusionKDiffusionPipeline

try:
       if not (is_torch_available() and is_transformers_available() and is_onnx_available()):
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_torch_and_transformers_and_onnx_objects import *  # noqa F403
else:
       from .pipelines import (
           OnnxStableDiffusionImgaImgPipeline,
           OnnxStableDiffusionInpaintPipeline,
           OnnxStableDiffusionInpaintPipelineLegacy,
           OnnxStableDiffusionPipeline,
           OnnxStableDiffusionUpscalePipeline,
           StableDiffusionOnnxPipeline,
       )

try:
       if not (is_torch_available() and is_librosa_available()):
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_torch_and_librosa_objects import *  # noqa F403
else:
       from .pipelines import AudioDiffusionPipeline, Mel

try:
       if not (is_transformers_available() and is_torch_available() and is_note_seq_available()):
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_transformers_and_torch_and_note_seq_objects import *  # noqa F403
else:
       from .pipelines import SpectrogramDiffusionPipeline

try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_flax_objects import *  # noqa F403
else:
       from .models.controlnet_flax import FlaxControlNetModel
       from .models.modeling_flax_utils import FlaxModelMixin
       from .models.unet_ad_condition_flax import FlaxUNetaDConditionModel
       from .models.vae_flax import FlaxAutoencoderKL
       from .pipelines import FlaxDiffusionPipeline
       from .schedulers import (
           FlaxDDIMScheduler,
           FlaxDDPMScheduler,
           FlaxDPMSolverMultistepScheduler,
           FlaxKarrasVeScheduler,
           FlaxLMSDiscreteScheduler,
           FlaxPNDMScheduler,
           FlaxSchedulerMixin,
           FlaxScoreSdeVeScheduler,
       )


try:
       if not (is_flax_available() and is_transformers_available()):
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_flax_and_transformers_objects import *  # noqa F403
else:
       from .pipelines import (
           FlaxStableDiffusionControlNetPipeline,
           FlaxStableDiffusionImgaImgPipeline,
           FlaxStableDiffusionInpaintPipeline,
           FlaxStableDiffusionPipeline,
       )

try:
       if not (is_note_seq_available()):
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       from .utils.dummy_note_seq_objects import *  # noqa F403
else:
       from .pipelines import MidiProcessor









import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class      __lowerCAmelCase     (		UpperCAmelCase_							,     unittest.TestCase      ):
					"""simple docstring"""






					A__				:		int			       =    LongformerTokenizer
					A__				:		Optional[int]			       =    True
					A__				:		Any			       =    LongformerTokenizerFast
					A__				:		Dict			       =    True



					def     _a							(     self					:       int					):






								"""simple docstring"""


								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								A__       =    [
								    'l',
								    'o',
								    'w',
								    'e',
								    'r',
								    's',
								    't',
								    'i',
								    'd',
								    'n',
								    '\u0120',
								    '\u0120l',
								    '\u0120n',
								    '\u0120lo',
								    '\u0120low',
								    'er',
								    '\u0120lowest',
								    '\u0120newer',
								    '\u0120wider',
								    '<unk>',
								]
								A__       =    dict(zip(_snake_case				,					range(len(_snake_case					)					)					)					)
								A__       =    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
								A__       =    {'unk_token': '<unk>'}

								A__       =    os.path.join(self.tmpdirname				,					VOCAB_FILES_NAMES['vocab_file']					)
								A__       =    os.path.join(self.tmpdirname				,					VOCAB_FILES_NAMES['merges_file']					)
								with open(self.vocab_file				,					'w'				,					encoding='utf-8'					) as fp:
											fp.write(json.dumps(_snake_case					) + '\n'					)
								with open(self.merges_file				,					'w'				,					encoding='utf-8'					) as fp:
											fp.write('\n'.join(_snake_case					)					)



					def     _a							(     self					:       int				,					**_snake_case					:       Union[str, Any]					):






								"""simple docstring"""


								kwargs.update(self.special_tokens_map					)
								return self.tokenizer_class.from_pretrained(self.tmpdirname				,					**_snake_case					)



					def     _a							(     self					:       Optional[int]				,					**_snake_case					:       List[Any]					):






								"""simple docstring"""


								kwargs.update(self.special_tokens_map					)
								return self.rust_tokenizer_class.from_pretrained(self.tmpdirname				,					**_snake_case					)



					def     _a							(     self					:       Any				,					_snake_case					:       Optional[Any]					):






								"""simple docstring"""


								A__       =    'lower newer'
								A__       =    'lower newer'
								return input_text, output_text



					def     _a							(     self					:       Any					):






								"""simple docstring"""


								A__       =    self.tokenizer_class(self.vocab_file				,					self.merges_file				,					**self.special_tokens_map					)
								A__       =    'lower newer'
								A__       =    ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
								A__       =    tokenizer.tokenize(_snake_case					)  # , add_prefix_space=True)
								self.assertListEqual(_snake_case				,					_snake_case					)

								A__       =    tokens + [tokenizer.unk_token]
								A__       =    [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case					)				,					_snake_case					)



					def     _a							(     self					:       List[str]					):






								"""simple docstring"""


								A__       =    self.get_tokenizer()

								self.assertListEqual(tokenizer.encode('Hello world!'				,					add_special_tokens=_snake_case					)				,					[0, 3_14_14, 2_32, 3_28, 2]					)
								self.assertListEqual(
								    tokenizer.encode('Hello world! cécé herlolip 418'				,					add_special_tokens=_snake_case					)				,					[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]				,					)



					@slow
					def     _a							(     self					:       List[Any]					):






								"""simple docstring"""


								A__       =    self.tokenizer_class.from_pretrained('allenai/longformer-base-4096'					)

								A__       =    tokenizer.encode('sequence builders'				,					add_special_tokens=_snake_case					)
								A__       =    tokenizer.encode('multi-sequence build'				,					add_special_tokens=_snake_case					)

								A__       =    tokenizer.encode(
								    'sequence builders'				,					add_special_tokens=_snake_case				,					add_prefix_space=_snake_case					)
								A__       =    tokenizer.encode(
								    'sequence builders'				,					'multi-sequence build'				,					add_special_tokens=_snake_case				,					add_prefix_space=_snake_case					)

								A__       =    tokenizer.build_inputs_with_special_tokens(_snake_case					)
								A__       =    tokenizer.build_inputs_with_special_tokens(_snake_case				,					_snake_case					)

								assert encoded_sentence == encoded_text_from_decode
								assert encoded_pair == encoded_pair_from_decode



					def     _a							(     self					:       List[str]					):






								"""simple docstring"""


								A__       =    self.get_tokenizer()

								A__       =    'Encode this sequence.'
								A__       =    tokenizer.byte_encoder[' '.encode('utf-8'					)[0]]

								# Testing encoder arguments
								A__       =    tokenizer.encode(_snake_case				,					add_special_tokens=_snake_case				,					add_prefix_space=_snake_case					)
								A__       =    tokenizer.convert_ids_to_tokens(encoded[0]					)[0]
								self.assertNotEqual(_snake_case				,					_snake_case					)

								A__       =    tokenizer.encode(_snake_case				,					add_special_tokens=_snake_case				,					add_prefix_space=_snake_case					)
								A__       =    tokenizer.convert_ids_to_tokens(encoded[0]					)[0]
								self.assertEqual(_snake_case				,					_snake_case					)

								tokenizer.add_special_tokens({'bos_token': '<s>'}					)
								A__       =    tokenizer.encode(_snake_case				,					add_special_tokens=_snake_case					)
								A__       =    tokenizer.convert_ids_to_tokens(encoded[1]					)[0]
								self.assertNotEqual(_snake_case				,					_snake_case					)

								# Testing spaces after special tokens
								A__       =    '<mask>'
								tokenizer.add_special_tokens(
								    {'mask_token': AddedToken(_snake_case				,					lstrip=_snake_case				,					rstrip=_snake_case					)}					)  # mask token has a left space
								A__       =    tokenizer.convert_tokens_to_ids(_snake_case					)

								A__       =    'Encode <mask> sequence'
								A__       =    'Encode <mask>sequence'

								A__       =    tokenizer.encode(_snake_case					)
								A__       =    encoded.index(_snake_case					)
								A__       =    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]					)[0]
								self.assertEqual(_snake_case				,					_snake_case					)

								A__       =    tokenizer.encode(_snake_case					)
								A__       =    encoded.index(_snake_case					)
								A__       =    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]					)[0]
								self.assertNotEqual(_snake_case				,					_snake_case					)



					def     _a							(     self					:       Dict					):






								"""simple docstring"""


								pass



					def     _a							(     self					:       Union[str, Any]					):






								"""simple docstring"""


								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
														A__       =    self.rust_tokenizer_class.from_pretrained(_snake_case				,					**_snake_case					)
														A__       =    self.tokenizer_class.from_pretrained(_snake_case				,					**_snake_case					)
														A__       =    'A, <mask> AllenNLP sentence.'
														A__       =    tokenizer_r.encode_plus(_snake_case				,					add_special_tokens=_snake_case				,					return_token_type_ids=_snake_case					)
														A__       =    tokenizer_p.encode_plus(_snake_case				,					add_special_tokens=_snake_case				,					return_token_type_ids=_snake_case					)

														# token_type_ids should put 0 everywhere
														self.assertEqual(sum(tokens_r['token_type_ids']					)				,					sum(tokens_p['token_type_ids']					)					)

														# attention_mask should put 1 everywhere, so sum over length should be 1
														self.assertEqual(
														    sum(tokens_r['attention_mask']					) / len(tokens_r['attention_mask']					)				,					sum(tokens_p['attention_mask']					) / len(tokens_p['attention_mask']					)				,					)

														A__       =    tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']					)
														A__       =    tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']					)

														# Rust correctly handles the space before the mask while python doesnt
														self.assertSequenceEqual(tokens_p['input_ids']				,					[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]					)
														self.assertSequenceEqual(tokens_r['input_ids']				,					[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]					)

														self.assertSequenceEqual(
														    _snake_case				,					['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']					)
														self.assertSequenceEqual(
														    _snake_case				,					['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']					)



					def     _a							(     self					:       List[Any]					):






								"""simple docstring"""


								for trim_offsets, add_prefix_space in itertools.product([True, False]				,					repeat=2					):
											A__       =    self.rust_tokenizer_class.from_pretrained(
											    self.tmpdirname				,					use_fast=_snake_case				,					add_prefix_space=_snake_case				,					trim_offsets=_snake_case					)

											A__       =    json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()					)
											A__       =    json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()					)

											self.assertEqual(pre_tokenizer_state['add_prefix_space']				,					_snake_case					)

											self.assertEqual(post_processor_state['add_prefix_space']				,					_snake_case					)
											self.assertEqual(post_processor_state['trim_offsets']				,					_snake_case					)



					def     _a							(     self					:       Optional[Any]					):






								"""simple docstring"""


								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
														A__       =    'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
														A__       =    F'''{text_of_1_token} {text_of_1_token}'''

														A__       =    self.rust_tokenizer_class.from_pretrained(
														    _snake_case				,					use_fast=_snake_case				,					add_prefix_space=_snake_case				,					trim_offsets=_snake_case					)
														A__       =    tokenizer_r(_snake_case				,					return_offsets_mapping=_snake_case				,					add_special_tokens=_snake_case					)
														self.assertEqual(encoding.offset_mapping[0]				,					(0, len(_snake_case					))					)
														self.assertEqual(
														    encoding.offset_mapping[1]				,					(len(_snake_case					) + 1, len(_snake_case					) + 1 + len(_snake_case					))				,					)

														A__       =    self.rust_tokenizer_class.from_pretrained(
														    _snake_case				,					use_fast=_snake_case				,					add_prefix_space=_snake_case				,					trim_offsets=_snake_case					)
														A__       =    tokenizer_r(_snake_case				,					return_offsets_mapping=_snake_case				,					add_special_tokens=_snake_case					)
														self.assertEqual(encoding.offset_mapping[0]				,					(0, len(_snake_case					))					)
														self.assertEqual(
														    encoding.offset_mapping[1]				,					(len(_snake_case					) + 1, len(_snake_case					) + 1 + len(_snake_case					))				,					)

														A__       =    self.rust_tokenizer_class.from_pretrained(
														    _snake_case				,					use_fast=_snake_case				,					add_prefix_space=_snake_case				,					trim_offsets=_snake_case					)
														A__       =    tokenizer_r(_snake_case				,					return_offsets_mapping=_snake_case				,					add_special_tokens=_snake_case					)
														self.assertEqual(encoding.offset_mapping[0]				,					(0, len(_snake_case					))					)
														self.assertEqual(
														    encoding.offset_mapping[1]				,					(len(_snake_case					), len(_snake_case					) + 1 + len(_snake_case					))				,					)

														A__       =    self.rust_tokenizer_class.from_pretrained(
														    _snake_case				,					use_fast=_snake_case				,					add_prefix_space=_snake_case				,					trim_offsets=_snake_case					)
														A__       =    tokenizer_r(_snake_case				,					return_offsets_mapping=_snake_case				,					add_special_tokens=_snake_case					)
														self.assertEqual(encoding.offset_mapping[0]				,					(0, len(_snake_case					))					)
														self.assertEqual(
														    encoding.offset_mapping[1]				,					(len(_snake_case					), len(_snake_case					) + 1 + len(_snake_case					))				,					)

														A__       =    F''' {text}'''

														# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
														#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
														# )
														# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
														# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
														# self.assertEqual(
														#     encoding.offset_mapping[1],
														#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
														# )

														A__       =    self.rust_tokenizer_class.from_pretrained(
														    _snake_case				,					use_fast=_snake_case				,					add_prefix_space=_snake_case				,					trim_offsets=_snake_case					)
														A__       =    tokenizer_r(_snake_case				,					return_offsets_mapping=_snake_case				,					add_special_tokens=_snake_case					)
														self.assertEqual(encoding.offset_mapping[0]				,					(1, 1 + len(_snake_case					))					)
														self.assertEqual(
														    encoding.offset_mapping[1]				,					(1 + len(_snake_case					) + 1, 1 + len(_snake_case					) + 1 + len(_snake_case					))				,					)

														A__       =    self.rust_tokenizer_class.from_pretrained(
														    _snake_case				,					use_fast=_snake_case				,					add_prefix_space=_snake_case				,					trim_offsets=_snake_case					)
														A__       =    tokenizer_r(_snake_case				,					return_offsets_mapping=_snake_case				,					add_special_tokens=_snake_case					)
														self.assertEqual(encoding.offset_mapping[0]				,					(0, 1 + len(_snake_case					))					)
														self.assertEqual(
														    encoding.offset_mapping[1]				,					(1 + len(_snake_case					), 1 + len(_snake_case					) + 1 + len(_snake_case					))				,					)

														A__       =    self.rust_tokenizer_class.from_pretrained(
														    _snake_case				,					use_fast=_snake_case				,					add_prefix_space=_snake_case				,					trim_offsets=_snake_case					)
														A__       =    tokenizer_r(_snake_case				,					return_offsets_mapping=_snake_case				,					add_special_tokens=_snake_case					)
														self.assertEqual(encoding.offset_mapping[0]				,					(0, 1 + len(_snake_case					))					)
														self.assertEqual(
														    encoding.offset_mapping[1]				,					(1 + len(_snake_case					), 1 + len(_snake_case					) + 1 + len(_snake_case					))				,					)




