






"""simple docstring"""



import copy
import inspect
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import TimesformerConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch
						from torch import nn

						from transformers import (
						    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
						    TimesformerForVideoClassification,
						    TimesformerModel,
						)
						from transformers.models.timesformer.modeling_timesformer import TIMESFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
						from transformers import VideoMAEImageProcessor






class 							__SCREAMING_SNAKE_CASE						:


							def __init__(			self      :						Dict			,    snake_case      :						Union[str, Any]			,    snake_case      :						List[str]=13			,    snake_case      :						Any=10			,    snake_case      :						Optional[Any]=3			,    snake_case      :						List[str]=2			,    snake_case      :						Union[str, Any]=2			,    snake_case      :						Tuple=True			,    snake_case      :						Union[str, Any]=True			,    snake_case      :						Optional[int]=32			,    snake_case      :						Optional[int]=5			,    snake_case      :						Any=4			,    snake_case      :						Tuple=37			,    snake_case      :						Union[str, Any]="gelu"			,    snake_case      :						Any=0.1			,    snake_case      :						Dict=0.1			,    snake_case      :						Any=10			,    snake_case      :						Optional[Any]=0.02			,    snake_case      :						Dict="divided_space_time"			,    snake_case      :						int=None			,    ):
													'''simple docstring'''





													A__						:  Dict		  =  parent
													A__						:  Dict		  =  batch_size
													A__						:  Optional[Any]		  =  image_size
													A__						:  Optional[Any]		  =  num_channels
													A__						:  Dict		  =  patch_size
													A__						:  int		  =  num_frames
													A__						:  Any		  =  is_training
													A__						:  str		  =  use_labels
													A__						:  List[str]		  =  hidden_size
													A__						:  List[str]		  =  num_hidden_layers
													A__						:  int		  =  num_attention_heads
													A__						:  List[Any]		  =  intermediate_size
													A__						:  str		  =  hidden_act
													A__						:  Union[str, Any]		  =  hidden_dropout_prob
													A__						:  Optional[Any]		  =  attention_probs_dropout_prob
													A__						:  Union[str, Any]		  =  attention_type
													A__						:  int		  =  initializer_range
													A__						:  Optional[int]		  =  scope
													A__						:  Any		  =  num_labels

													# in TimeSformer, the number of spatial tokens equals num_frames * num_patches per frame + 1 CLS token
													A__						:  str		  =  (image_size // patch_size) ** 2
													A__						:  List[Any]		  =  (num_frames) * self.num_patches_per_frame + 1


							def 						_UpperCamelCase       (			self      :						Dict							):
													'''simple docstring'''





													A__						:  List[str]		  =  floats_tensor(
													    [self.batch_size, self.num_frames, self.num_channels, self.image_size, self.image_size]							)

													A__						:  Any		  =  None
													if self.use_labels:
																			A__						:  List[Any]		  =  ids_tensor([self.batch_size]			,    self.num_labels							)

													A__						:  Optional[Any]		  =  self.get_config()

													return config, pixel_values, labels


							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  int		  =  TimesformerConfig(
													    image_size=self.image_size			,    patch_size=self.patch_size			,    num_channels=self.num_channels			,    num_frames=self.num_frames			,    hidden_size=self.hidden_size			,    num_hidden_layers=self.num_hidden_layers			,    num_attention_heads=self.num_attention_heads			,    intermediate_size=self.intermediate_size			,    hidden_act=self.hidden_act			,    hidden_dropout_prob=self.hidden_dropout_prob			,    attention_probs_dropout_prob=self.attention_probs_dropout_prob			,    initializer_range=self.initializer_range			,    attention_type=self.attention_type			,    )
													A__						:  List[str]		  =  self.num_labels
													return config


							def 						_UpperCamelCase       (			self      :						Any			,    snake_case      :						List[Any]			,    snake_case      :						Dict			,    snake_case      :						Tuple							):
													'''simple docstring'''





													A__						:  List[str]		  =  TimesformerModel(config=_A							)
													model.to(_A							)
													model.eval()
													A__						:  int		  =  model(_A							)
													self.parent.assertEqual(result.last_hidden_state.shape			,    (self.batch_size, self.seq_length, self.hidden_size)							)


							def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						Optional[int]			,    snake_case      :						Union[str, Any]			,    snake_case      :						Optional[int]							):
													'''simple docstring'''





													A__						:  Dict		  =  TimesformerForVideoClassification(_A							)
													model.to(_A							)
													model.eval()

													A__						:  Optional[Any]		  =  model(_A							)

													# verify the logits shape
													A__						:  List[Any]		  =  torch.Size((self.batch_size, self.num_labels)							)
													self.parent.assertEqual(result.logits.shape			,    _A							)


							def 						_UpperCamelCase       (			self      :						Optional[Any]							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  self.prepare_config_and_inputs()
													A__						:  int		  =  config_and_inputs
													A__						:  Optional[int]		  =  {'pixel_values': pixel_values}
													return config, inputs_dict






@require_torch
class 							__SCREAMING_SNAKE_CASE						(				a__       ,		a__       ,		unittest.TestCase		):
							snake_case_     =							(TimesformerModel, TimesformerForVideoClassification) if is_torch_available() else ()
							snake_case_     =							(
							    {'feature-extraction': TimesformerModel, 'video-classification': TimesformerForVideoClassification}
							    if is_torch_available()
							    else {}
							)

							snake_case_     =							False
							snake_case_     =							False
							snake_case_     =							False
							snake_case_     =							False


							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													A__						:  List[str]		  =  TimesformerModelTester(self							)
													A__						:  Optional[int]		  =  ConfigTester(
													    self			,    config_class=_A			,    has_text_modality=_A			,    hidden_size=37							)


							def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						str			,    snake_case      :						List[str]			,    snake_case      :						List[str]=False							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  copy.deepcopy(_A							)

													if return_labels:
																			if model_class in get_values(_A							):
																									A__						:  int		  =  torch.zeros(
																									    self.model_tester.batch_size			,    dtype=torch.long			,    device=_A							)

													return inputs_dict


							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													self.config_tester.run_common_tests()


							@unittest.skip(reason="""TimeSformer does not use inputs_embeds"""							)
							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													pass


							def 						_UpperCamelCase       (			self      :						List[Any]							):
													'''simple docstring'''





													A__						:  Any		  =  self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																			A__						:  Union[str, Any]		  =  model_class(_A							)
																			self.assertIsInstance(model.get_input_embeddings()			,    (nn.Module)							)
																			A__						:  Optional[Any]		  =  model.get_output_embeddings()
																			self.assertTrue(x is None or isinstance(_A			,    nn.Linear							)							)


							def 						_UpperCamelCase       (			self      :						str							):
													'''simple docstring'''





													A__						:  Dict		  =  self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																			A__						:  Any		  =  model_class(_A							)
																			A__						:  Optional[Any]		  =  inspect.signature(model.forward							)
																			# signature.parameters is an OrderedDict => so arg_names order is deterministic
																			A__						:  Optional[Any]		  =  [*signature.parameters.keys()]

																			A__						:  Any		  =  ['pixel_values']
																			self.assertListEqual(arg_names[:1]			,    _A							)


							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  List[str]		  =  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*_A							)


							def 						_UpperCamelCase       (			self      :						List[Any]							):
													'''simple docstring'''





													A__						:  Optional[int]		  =  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_video_classification(*_A							)


							@slow
							def 						_UpperCamelCase       (			self      :						str							):
													'''simple docstring'''





													for model_name in TIMESFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			A__						:  Any		  =  TimesformerModel.from_pretrained(_A							)
																			self.assertIsNotNone(_A							)


							def 						_UpperCamelCase       (			self      :						str							):
													'''simple docstring'''





													if not self.has_attentions:
																			pass

													else:
																			A__						:  Dict		  =  self.model_tester.prepare_config_and_inputs_for_common()
																			A__						:  List[str]		  =  True

																			for model_class in self.all_model_classes:
																									A__						:  List[str]		  =  self.model_tester.seq_length
																									A__						:  Union[str, Any]		  =  self.model_tester.num_frames

																									A__						:  Optional[int]		  =  True
																									A__						:  str		  =  False
																									A__						:  Union[str, Any]		  =  True
																									A__						:  List[str]		  =  model_class(_A							)
																									model.to(_A							)
																									model.eval()
																									with torch.no_grad():
																															A__						:  int		  =  model(**self._prepare_for_class(_A			,    _A							)							)
																									A__						:  Dict		  =  outputs.attentions
																									self.assertEqual(len(_A							)			,    self.model_tester.num_hidden_layers							)

																									# check that output_attentions also work using config
																									del inputs_dict["output_attentions"]
																									A__						:  List[Any]		  =  True
																									A__						:  str		  =  model_class(_A							)
																									model.to(_A							)
																									model.eval()
																									with torch.no_grad():
																															A__						:  int		  =  model(**self._prepare_for_class(_A			,    _A							)							)
																									A__						:  str		  =  outputs.attentions
																									self.assertEqual(len(_A							)			,    self.model_tester.num_hidden_layers							)

																									# attentions has shape (batch_size x num_frames) x num_heads x (num_patches per frame + 1) x (num_patches per frame + 1)
																									self.assertListEqual(
																									    list(attentions[0].shape[-3:]							)			,    [self.model_tester.num_attention_heads, seq_len // num_frames + 1, seq_len // num_frames + 1]			,    )
																									A__						:  Optional[Any]		  =  len(_A							)

																									# Check attention is always last and order is fine
																									A__						:  int		  =  True
																									A__						:  List[str]		  =  True
																									A__						:  Dict		  =  model_class(_A							)
																									model.to(_A							)
																									model.eval()
																									with torch.no_grad():
																															A__						:  str		  =  model(**self._prepare_for_class(_A			,    _A							)							)

																									self.assertEqual(out_len + 1			,    len(_A							)							)

																									A__						:  Tuple		  =  outputs.attentions

																									self.assertEqual(len(_A							)			,    self.model_tester.num_hidden_layers							)

																									# attentions has shape (batch_size x num_frames) x num_heads x (num_patches per frame + 1) x (num_patches per frame + 1)
																									self.assertListEqual(
																									    list(self_attentions[0].shape[-3:]							)			,    [self.model_tester.num_attention_heads, seq_len // num_frames + 1, seq_len // num_frames + 1]			,    )


							def 						_UpperCamelCase       (			self      :						Optional[Any]							):
													'''simple docstring'''





													def check_hidden_states_output(snake_case      :						List[Any]			,    snake_case      :						Optional[Any]			,    snake_case      :						Optional[int]							):
																			A__						:  Union[str, Any]		  =  model_class(_A							)
																			model.to(_A							)
																			model.eval()

																			with torch.no_grad():
																									A__						:  Union[str, Any]		  =  model(**self._prepare_for_class(_A			,    _A							)							)

																			A__						:  List[Any]		  =  outputs.hidden_states
																			A__						:  int		  =  self.model_tester.num_hidden_layers + 1
																			self.assertEqual(len(_A							)			,    _A							)

																			A__						:  Any		  =  self.model_tester.seq_length

																			self.assertListEqual(
																			    list(hidden_states[0].shape[-2:]							)			,    [seq_length, self.model_tester.hidden_size]			,    )

													A__						:  int		  =  self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																			A__						:  Optional[Any]		  =  True
																			check_hidden_states_output(_A			,    _A			,    _A							)

																			# check that output_hidden_states also work using config
																			del inputs_dict["output_hidden_states"]
																			A__						:  str		  =  True

																			check_hidden_states_output(_A			,    _A			,    _A							)




def      _lowerCAmelCase		(							)		->Any:
						A__						:  Union[str, Any]		  =  hf_hub_download(
						    repo_id="""hf-internal-testing/spaghetti-video""",						filename="""eating_spaghetti.npy""",						repo_type="""dataset"""      )
						A__						:  Any		  =  np.load(_lowerCamelCase      )
						return list(_lowerCamelCase      )







@require_torch
@require_vision
class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):


							@cached_property
							def 						_UpperCamelCase       (			self      :						Tuple							):
													'''simple docstring'''





													return (
													    VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]			,    image_std=[0.5, 0.5, 0.5]							)
													    if is_vision_available()
													    else None
													)


							@slow
							def 						_UpperCamelCase       (			self      :						str							):
													'''simple docstring'''





													A__						:  Dict		  =  TimesformerForVideoClassification.from_pretrained("""facebook/timesformer-base-finetuned-k400"""							).to(
													    _A							)

													A__						:  List[Any]		  =  self.default_image_processor
													A__						:  Tuple		  =  prepare_video()
													A__						:  int		  =  image_processor(video[:8]			,    return_tensors="""pt"""							).to(_A							)

													# forward pass
													with torch.no_grad():
																			A__						:  Optional[int]		  =  model(**_A							)

													# verify the logits
													A__						:  Optional[int]		  =  torch.Size((1, 400)							)
													self.assertEqual(outputs.logits.shape			,    _A							)

													A__						:  Optional[Any]		  =  torch.tensor([-0.3016, -0.7713, -0.4205]							).to(_A							)

													self.assertTrue(torch.allclose(outputs.logits[0, :3]			,    _A			,    atol=1e-4							)							)














"""simple docstring"""



import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A_    =							logging.get_logger(__name__)


A_    =							{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

A_    =							{
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
}

A_    =							{
    '''allenai/longformer-base-4096''': 4096,
    '''allenai/longformer-large-4096''': 4096,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 4096,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 4096,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 4096,
}




@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      _lowerCAmelCase		(							)		->List[str]:
						A__						:  Tuple		  =  (
						    list(range(ord("""!"""      ),						ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      ),						ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      ),						ord("""ÿ"""      ) + 1      )      )
						)
						A__						:  int		  =  bs[:]
						A__						:  Optional[Any]		  =  0
						for b in range(2**8      ):
												if b not in bs:
																		bs.append(UpperCAmelCase__      )
																		cs.append(2**8 + n      )
																		n += 1
						A__						:  Optional[Any]		  =  [chr(UpperCAmelCase__      ) for n in cs]
						return dict(zip(UpperCAmelCase__,						UpperCAmelCase__      )      )




def      _lowerCAmelCase		(							UpperCAmelCase__       :					int      )		->Tuple:
						A__						:  Tuple		  =  set()
						A__						:  Tuple		  =  word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)      )
												A__						:  str		  =  char
						return pairs






class 							__SCREAMING_SNAKE_CASE						(				UpperCamelCase		):
							snake_case_     =							VOCAB_FILES_NAMES
							snake_case_     =							PRETRAINED_VOCAB_FILES_MAP
							snake_case_     =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case_     =							['input_ids', 'attention_mask']


							def __init__(			self      :						List[str]			,    snake_case      :						List[str]			,    snake_case      :						str			,    snake_case      :						Dict="replace"			,    snake_case      :						Optional[int]="<s>"			,    snake_case      :						Any="</s>"			,    snake_case      :						List[Any]="</s>"			,    snake_case      :						str="<s>"			,    snake_case      :						Tuple="<unk>"			,    snake_case      :						List[str]="<pad>"			,    snake_case      :						Optional[Any]="<mask>"			,    snake_case      :						Tuple=False			,    **snake_case      :						Dict			,    ):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else bos_token
													A__						:  Optional[Any]		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else eos_token
													A__						:  int		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else sep_token
													A__						:  str		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else cls_token
													A__						:  Optional[Any]		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else unk_token
													A__						:  Optional[int]		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else pad_token

													# Mask token behave like a normal word, i.e. include the space before it
													A__						:  Union[str, Any]		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else mask_token

													super().__init__(
													    errors=snake_case			,    bos_token=snake_case			,    eos_token=snake_case			,    unk_token=snake_case			,    sep_token=snake_case			,    cls_token=snake_case			,    pad_token=snake_case			,    mask_token=snake_case			,    add_prefix_space=snake_case			,    **snake_case			,    )

													with open(snake_case			,    encoding="""utf-8"""							) as vocab_handle:
																			A__						:  Tuple		  =  json.load(snake_case							)
													A__						:  Dict		  =  {v: k for k, v in self.encoder.items()}
													A__						:  Dict		  =  errors  # how to handle errors in decoding
													A__						:  Dict		  =  bytes_to_unicode()
													A__						:  Optional[int]		  =  {v: k for k, v in self.byte_encoder.items()}
													with open(snake_case			,    encoding="""utf-8"""							) as merges_handle:
																			A__						:  List[str]		  =  merges_handle.read().split("""\n"""							)[1:-1]
													A__						:  Dict		  =  [tuple(merge.split()							) for merge in bpe_merges]
													A__						:  Union[str, Any]		  =  dict(zip(snake_case			,    range(len(snake_case							)							)							)							)
													A__						:  Tuple		  =  {}
													A__						:  Optional[int]		  =  add_prefix_space

													# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
													A__						:  Union[str, Any]		  =  re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""							)


							@property
							def 						_UpperCamelCase       (			self      :						List[Any]							):
													'''simple docstring'''





													return len(self.encoder							)


							def 						_UpperCamelCase       (			self      :						List[Any]							):
													'''simple docstring'''





													return dict(self.encoder			,    **self.added_tokens_encoder							)


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						Tuple							):
													'''simple docstring'''





													if token in self.cache:
																			return self.cache[token]
													A__						:  str		  =  tuple(snake_case							)
													A__						:  Any		  =  get_pairs(snake_case							)

													if not pairs:
																			return token

													while True:
																			A__						:  int		  =  min(snake_case			,    key=lambda snake_case							: self.bpe_ranks.get(snake_case			,    float("""inf"""							)							)							)
																			if bigram not in self.bpe_ranks:
																									break
																			A__  ,  A__						:  int		  =  bigram
																			A__						:  List[str]		  =  []
																			A__						:  Dict		  =  0
																			while i < len(snake_case							):
																									try:
																															A__						:  int		  =  word.index(snake_case			,    snake_case							)
																									except ValueError:
																															new_word.extend(word[i:]							)
																															break
																									else:
																															new_word.extend(word[i:j]							)
																															A__						:  str		  =  j

																									if word[i] == first and i < len(snake_case							) - 1 and word[i + 1] == second:
																															new_word.append(first + second							)
																															i += 2
																									else:
																															new_word.append(word[i]							)
																															i += 1
																			A__						:  Any		  =  tuple(snake_case							)
																			A__						:  Optional[Any]		  =  new_word
																			if len(snake_case							) == 1:
																									break
																			else:
																									A__						:  int		  =  get_pairs(snake_case							)
													A__						:  Dict		  =  """ """.join(snake_case							)
													A__						:  int		  =  word
													return word


							def 						_UpperCamelCase       (			self      :						int			,    snake_case      :						List[str]							):
													'''simple docstring'''





													A__						:  int		  =  []
													for token in re.findall(self.pat			,    snake_case							):
																			A__						:  Union[str, Any]		  =  """""".join(
																			    self.byte_encoder[b] for b in token.encode("""utf-8"""							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																			bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case							).split(""" """							)							)
													return bpe_tokens


							def 						_UpperCamelCase       (			self      :						Optional[Any]			,    snake_case      :						Dict							):
													'''simple docstring'''





													return self.encoder.get(snake_case			,    self.encoder.get(self.unk_token							)							)


							def 						_UpperCamelCase       (			self      :						Optional[int]			,    snake_case      :						int							):
													'''simple docstring'''





													return self.decoder.get(snake_case							)


							def 						_UpperCamelCase       (			self      :						Union[str, Any]			,    snake_case      :						Union[str, Any]							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  """""".join(snake_case							)
													A__						:  Optional[Any]		  =  bytearray([self.byte_decoder[c] for c in text]							).decode("""utf-8"""			,    errors=self.errors							)
													return text


							def 						_UpperCamelCase       (			self      :						List[str]			,    snake_case      :						str			,    snake_case      :						Optional[str] = None							):
													'''simple docstring'''





													if not os.path.isdir(snake_case							):
																			logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
																			return
													A__						:  Any		  =  os.path.join(
													    snake_case			,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
													A__						:  int		  =  os.path.join(
													    snake_case			,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]							)

													with open(snake_case			,    """w"""			,    encoding="""utf-8"""							) as f:
																			f.write(json.dumps(self.encoder			,    indent=2			,    sort_keys=snake_case			,    ensure_ascii=snake_case							) + """\n"""							)

													A__						:  Optional[int]		  =  0
													with open(snake_case			,    """w"""			,    encoding="""utf-8"""							) as writer:
																			writer.write("""#version: 0.2\n"""							)
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,    key=lambda snake_case							: kv[1]							):
																									if index != token_index:
																															logger.warning(
																															    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																															    """ Please check that the tokenizer is not corrupted!"""							)
																															A__						:  Dict		  =  token_index
																									writer.write(""" """.join(snake_case							) + """\n"""							)
																									index += 1

													return vocab_file, merge_file


							def 						_UpperCamelCase       (			self      :						List[Any]			,    snake_case      :						List[int]			,    snake_case      :						Optional[List[int]] = None							):
													'''simple docstring'''





													if token_ids_a is None:
																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
													A__						:  Optional[int]		  =  [self.cls_token_id]
													A__						:  int		  =  [self.sep_token_id]
													return cls + token_ids_a + sep + sep + token_ids_a + sep


							def 						_UpperCamelCase       (			self      :						int			,    snake_case      :						List[int]			,    snake_case      :						Optional[List[int]] = None			,    snake_case      :						bool = False							):
													'''simple docstring'''





													if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=snake_case			,    token_ids_a=snake_case			,    already_has_special_tokens=snake_case							)

													if token_ids_a is None:
																			return [1] + ([0] * len(snake_case							)) + [1]
													return [1] + ([0] * len(snake_case							)) + [1, 1] + ([0] * len(snake_case							)) + [1]


							def 						_UpperCamelCase       (			self      :						Union[str, Any]			,    snake_case      :						List[int]			,    snake_case      :						Optional[List[int]] = None							):
													'''simple docstring'''





													A__						:  int		  =  [self.sep_token_id]
													A__						:  int		  =  [self.cls_token_id]

													if token_ids_a is None:
																			return len(cls + token_ids_a + sep							) * [0]
													return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						Optional[Any]			,    snake_case      :						List[str]=False			,    **snake_case      :						Optional[Any]							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  kwargs.pop("""add_prefix_space"""			,    self.add_prefix_space							)
													if (is_split_into_words or add_prefix_space) and (len(snake_case							) > 0 and not text[0].isspace()):
																			A__						:  str		  =  """ """ + text
													return (text, kwargs)






