



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation

SCREAMING_SNAKE_CASE__       =      logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__       =      {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

SCREAMING_SNAKE_CASE__       =      {
    '''vocab_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'''},
    '''merges_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'''},
    '''tokenizer_config_file''': {
        '''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'''
    },
}

SCREAMING_SNAKE_CASE__       =      {'''facebook/blenderbot-3B''': 1_2_8}


@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def A						(       )						->      int:
			A__       =    (
			    list(range(ord('!'					) ,	ord('~'					) + 1					)					) + list(range(ord('¡'					) ,	ord('¬'					) + 1					)					) + list(range(ord('®'					) ,	ord('ÿ'					) + 1					)					)
			)
			A__       =    bs[:]
			A__       =    0
			for b in range(2**8					):
						if b not in bs:
									bs.append(__UpperCamelCase					)
									cs.append(2**8 + n					)
									n += 1
			A__       =    [chr(__UpperCamelCase					) for n in cs]
			return dict(zip(__UpperCamelCase ,	__UpperCamelCase					)					)


def A						(       __UpperCamelCase					)						->      Union[str, Any]:
			A__       =    set()
			A__       =    word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)					)
						A__       =    char
			return pairs






class      __lowerCAmelCase     (		UpperCAmelCase_      ):
					"""simple docstring"""






					A__				:		Union[str, Any]			       =    VOCAB_FILES_NAMES
					A__				:		Dict			       =    PRETRAINED_VOCAB_FILES_MAP
					A__				:		Any			       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__				:		List[str]			       =    ["input_ids", "attention_mask"]



					def __init__(     self					:       Tuple				,					_snake_case					:       str				,					_snake_case					:       Tuple				,					_snake_case					:       int="replace"				,					_snake_case					:       List[str]="<s>"				,					_snake_case					:       List[Any]="</s>"				,					_snake_case					:       Optional[Any]="</s>"				,					_snake_case					:       List[str]="<s>"				,					_snake_case					:       int="<unk>"				,					_snake_case					:       List[str]="<pad>"				,					_snake_case					:       int="<mask>"				,					_snake_case					:       Optional[int]=False				,					**_snake_case					:       int				,					):






								"""simple docstring"""


								A__       =    AddedToken(_snake_case				,					lstrip=_snake_case				,					rstrip=_snake_case					) if isinstance(_snake_case				,					_snake_case					) else bos_token
								A__       =    AddedToken(_snake_case				,					lstrip=_snake_case				,					rstrip=_snake_case					) if isinstance(_snake_case				,					_snake_case					) else eos_token
								A__       =    AddedToken(_snake_case				,					lstrip=_snake_case				,					rstrip=_snake_case					) if isinstance(_snake_case				,					_snake_case					) else sep_token
								A__       =    AddedToken(_snake_case				,					lstrip=_snake_case				,					rstrip=_snake_case					) if isinstance(_snake_case				,					_snake_case					) else cls_token
								A__       =    AddedToken(_snake_case				,					lstrip=_snake_case				,					rstrip=_snake_case					) if isinstance(_snake_case				,					_snake_case					) else unk_token
								A__       =    AddedToken(_snake_case				,					lstrip=_snake_case				,					rstrip=_snake_case					) if isinstance(_snake_case				,					_snake_case					) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								A__       =    AddedToken(_snake_case				,					lstrip=_snake_case				,					rstrip=_snake_case					) if isinstance(_snake_case				,					_snake_case					) else mask_token

								super().__init__(
								    errors=_snake_case				,					bos_token=_snake_case				,					eos_token=_snake_case				,					unk_token=_snake_case				,					sep_token=_snake_case				,					cls_token=_snake_case				,					pad_token=_snake_case				,					mask_token=_snake_case				,					add_prefix_space=_snake_case				,					**_snake_case				,					)

								with open(_snake_case				,					encoding='utf-8'					) as vocab_handle:
											A__       =    json.load(_snake_case					)
								A__       =    {v: k for k, v in self.encoder.items()}
								A__       =    errors  # how to handle errors in decoding
								A__       =    bytes_to_unicode()
								A__       =    {v: k for k, v in self.byte_encoder.items()}
								with open(_snake_case				,					encoding='utf-8'					) as merges_handle:
											A__       =    merges_handle.read().split('\n'					)[1:-1]
								A__       =    [tuple(merge.split()					) for merge in bpe_merges]
								A__       =    dict(zip(_snake_case				,					range(len(_snake_case					)					)					)					)
								A__       =    {}
								A__       =    add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								A__       =    re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'					)



					@property
					# Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
					def     _a							(     self					:       int					):






								"""simple docstring"""


								return len(self.encoder					)



					def     _a							(     self					:       List[Any]					):






								"""simple docstring"""


								return dict(self.encoder				,					**self.added_tokens_encoder					)



					def     _a							(     self					:       Optional[Any]				,					_snake_case					:       Any					):






								"""simple docstring"""


								if token in self.cache:
											return self.cache[token]
								A__       =    tuple(_snake_case					)
								A__       =    get_pairs(_snake_case					)

								if not pairs:
											return token

								while True:
											A__       =    min(_snake_case				,					key=lambda _snake_case					: self.bpe_ranks.get(_snake_case				,					float('inf'					)					)					)
											if bigram not in self.bpe_ranks:
														break
											A__     , A__       =    bigram
											A__       =    []
											A__       =    0
											while i < len(_snake_case					):
														try:
																	A__       =    word.index(_snake_case				,					_snake_case					)
														except ValueError:
																	new_word.extend(word[i:]					)
																	break
														else:
																	new_word.extend(word[i:j]					)
																	A__       =    j

														if word[i] == first and i < len(_snake_case					) - 1 and word[i + 1] == second:
																	new_word.append(first + second					)
																	i += 2
														else:
																	new_word.append(word[i]					)
																	i += 1
											A__       =    tuple(_snake_case					)
											A__       =    new_word
											if len(_snake_case					) == 1:
														break
											else:
														A__       =    get_pairs(_snake_case					)
								A__       =    ' '.join(_snake_case					)
								A__       =    word
								return word



					def     _a							(     self					:       int				,					_snake_case					:       Dict					):






								"""simple docstring"""


								A__       =    []
								for token in re.findall(self.pat				,					_snake_case					):
											A__       =    ''.join(
											    self.byte_encoder[b] for b in token.encode('utf-8'					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_snake_case					).split(' '					)					)
								return bpe_tokens



					def     _a							(     self					:       Any				,					_snake_case					:       List[str]					):






								"""simple docstring"""


								return self.encoder.get(_snake_case				,					self.encoder.get(self.unk_token					)					)



					def     _a							(     self					:       str				,					_snake_case					:       Optional[Any]					):






								"""simple docstring"""


								return self.decoder.get(_snake_case					)



					def     _a							(     self					:       Optional[int]				,					_snake_case					:       List[str]					):






								"""simple docstring"""


								A__       =    ''.join(_snake_case					)
								A__       =    bytearray([self.byte_decoder[c] for c in text]					).decode('utf-8'				,					errors=self.errors					)
								return text



					def     _a							(     self					:       List[str]				,					_snake_case					:       str				,					_snake_case					:       Optional[str] = None					):






								"""simple docstring"""


								if not os.path.isdir(_snake_case					):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
											return
								A__       =    os.path.join(
								    _snake_case				,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
								A__       =    os.path.join(
								    _snake_case				,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']					)

								with open(_snake_case				,					'w'				,					encoding='utf-8'					) as f:
											f.write(json.dumps(self.encoder				,					indent=2				,					sort_keys=_snake_case				,					ensure_ascii=_snake_case					) + '\n'					)

								A__       =    0
								with open(_snake_case				,					'w'				,					encoding='utf-8'					) as writer:
											writer.write('#version: 0.2\n'					)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,					key=lambda _snake_case					: kv[1]					):
														if index != token_index:
																	logger.warning(
																	    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																	    ' Please check that the tokenizer is not corrupted!'					)
																	A__       =    token_index
														writer.write(' '.join(_snake_case					) + '\n'					)
														index += 1

								return vocab_file, merge_file



					def     _a							(     self					:       Union[str, Any]				,					_snake_case					:       List[int]				,					_snake_case					:       Optional[List[int]] = None				,					_snake_case					:       bool = False					):






								"""simple docstring"""


								if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_snake_case				,					token_ids_a=_snake_case				,					already_has_special_tokens=_snake_case					)

								if token_ids_a is None:
											return [1] + ([0] * len(_snake_case					)) + [1]
								return [1] + ([0] * len(_snake_case					)) + [1, 1] + ([0] * len(_snake_case					)) + [1]



					def     _a							(     self					:       str				,					_snake_case					:       List[int]				,					_snake_case					:       Optional[List[int]] = None					):






								"""simple docstring"""


								A__       =    [self.sep_token_id]
								A__       =    [self.cls_token_id]

								if token_ids_a is None:
											return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]



					def     _a							(     self					:       Optional[int]				,					_snake_case					:       Optional[Any]				,					_snake_case					:       str=False				,					**_snake_case					:       int					):






								"""simple docstring"""


								A__       =    kwargs.pop('add_prefix_space'				,					self.add_prefix_space					)
								if (is_split_into_words or add_prefix_space) and (len(_snake_case					) > 0 and not text[0].isspace()):
											A__       =    ' ' + text
								return (text, kwargs)



					def     _a							(     self					:       str				,					_snake_case					:       List[int]				,					_snake_case					:       Optional[List[int]] = None					):






								"""simple docstring"""


								return token_ids_a + [self.eos_token_id]



					def     _a							(     self					:       List[Any]				,					_snake_case					:       "Conversation"					):






								"""simple docstring"""


								A__       =    []
								for is_user, text in conversation.iter_texts():
											if is_user:
														# We need to space prefix as it's being done within blenderbot
														inputs.append(' ' + text					)
											else:
														# Generated responses should contain them already.
														inputs.append(_snake_case					)

								A__       =    '  '.join(_snake_case					)
								A__       =    self.encode(_snake_case					)
								if len(_snake_case					) > self.model_max_length:
											A__       =    input_ids[-self.model_max_length :]
											logger.warning(F'''Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'''					)
								return input_ids









import inspect
import unittest

from transformers import ViTHybridConfig
from transformers.testing_utils import require_accelerate, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch
							from torch import nn

							from transformers import ViTHybridForImageClassification, ViTHybridImageProcessor, ViTHybridModel
							from transformers.models.vit_hybrid.modeling_vit_hybrid import VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
							from PIL import Image


class      __lowerCAmelCase     :
					"""simple docstring"""






					def __init__(     self					:       List[Any]				,					_snake_case					:       Any				,					_snake_case					:       Optional[int]=13				,					_snake_case					:       Optional[Any]=64				,					_snake_case					:       List[str]=2				,					_snake_case					:       Any=3				,					_snake_case					:       Union[str, Any]=True				,					_snake_case					:       Dict=True				,					_snake_case					:       int=32				,					_snake_case					:       int=5				,					_snake_case					:       Union[str, Any]=4				,					_snake_case					:       int=37				,					_snake_case					:       Tuple="gelu"				,					_snake_case					:       Optional[int]=0.1				,					_snake_case					:       Dict=0.1				,					_snake_case					:       List[str]=10				,					_snake_case					:       Union[str, Any]=0.02				,					_snake_case					:       Dict=[1, 16, 4, 4]				,					_snake_case					:       Dict=None				,					):






								"""simple docstring"""


								A__       =    parent
								A__       =    batch_size
								A__       =    image_size
								A__       =    patch_size
								A__       =    num_channels
								A__       =    is_training
								A__       =    use_labels
								A__       =    hidden_size
								A__       =    num_hidden_layers
								A__       =    num_attention_heads
								A__       =    intermediate_size
								A__       =    hidden_act
								A__       =    hidden_dropout_prob
								A__       =    attention_probs_dropout_prob
								A__       =    type_sequence_label_size
								A__       =    initializer_range
								A__       =    scope
								A__       =    backbone_featmap_shape

								# in ViT hybrid, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
								# the number of patches is based on the feature map of the backbone, which by default uses an output stride
								# of 32, which means that the feature map has a spatial resolution of 1/32 of the input image size
								A__       =    (self.image_size // 32) ** 2
								A__       =    num_patches + 1



					def     _a							(     self					:       Any					):






								"""simple docstring"""


								A__       =    floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]					)

								A__       =    None
								if self.use_labels:
											A__       =    ids_tensor([self.batch_size]				,					self.type_sequence_label_size					)

								A__       =    self.get_config()

								return config, pixel_values, labels



					def     _a							(     self					:       Tuple					):






								"""simple docstring"""


								A__       =    {
								    'global_padding': 'same',
								    'layer_type': 'bottleneck',
								    'depths': [3, 4, 9],
								    'out_features': ['stage1', 'stage2', 'stage3'],
								    'embedding_dynamic_padding': True,
								    'hidden_sizes': [4, 8, 16, 32],
								    'num_groups': 2,
								}

								return ViTHybridConfig(
								    image_size=self.image_size				,					patch_size=self.patch_size				,					num_channels=self.num_channels				,					hidden_size=self.hidden_size				,					num_hidden_layers=self.num_hidden_layers				,					num_attention_heads=self.num_attention_heads				,					intermediate_size=self.intermediate_size				,					hidden_act=self.hidden_act				,					hidden_dropout_prob=self.hidden_dropout_prob				,					attention_probs_dropout_prob=self.attention_probs_dropout_prob				,					is_decoder=_snake_case				,					initializer_range=self.initializer_range				,					backbone_featmap_shape=self.backbone_featmap_shape				,					backbone_config=_snake_case				,					)



					def     _a							(     self					:       int				,					_snake_case					:       Optional[int]				,					_snake_case					:       Union[str, Any]				,					_snake_case					:       Optional[int]					):






								"""simple docstring"""


								A__       =    ViTHybridModel(config=_snake_case					)
								model.to(_snake_case					)
								model.eval()
								A__       =    model(_snake_case					)
								self.parent.assertEqual(result.last_hidden_state.shape				,					(self.batch_size, self.seq_length, self.hidden_size)					)



					def     _a							(     self					:       List[str]				,					_snake_case					:       str				,					_snake_case					:       Union[str, Any]				,					_snake_case					:       Any					):






								"""simple docstring"""


								A__       =    self.type_sequence_label_size
								A__       =    ViTHybridForImageClassification(_snake_case					)
								model.to(_snake_case					)
								model.eval()
								A__       =    model(_snake_case				,					labels=_snake_case					)
								self.parent.assertEqual(result.logits.shape				,					(self.batch_size, self.type_sequence_label_size)					)



					def     _a							(     self					:       Dict					):






								"""simple docstring"""


								A__       =    self.prepare_config_and_inputs()
								A__     , A__     , A__       =    config_and_inputs
								A__       =    {'pixel_values': pixel_values}
								return config, inputs_dict







@require_torch
class      __lowerCAmelCase     (		UpperCAmelCase_							,     UpperCAmelCase_							,     unittest.TestCase      ):
					"""simple docstring"""






					A__				:		Union[str, Any]			       =    (ViTHybridModel, ViTHybridForImageClassification) if is_torch_available() else ()
					A__				:		str			       =    (
					    {"feature-extraction": ViTHybridModel, "image-classification": ViTHybridForImageClassification}
					    if is_torch_available()
					    else {}
					)
					A__				:		Union[str, Any]			       =    False
					A__				:		Any			       =    False
					A__				:		Union[str, Any]			       =    False



					def     _a							(     self					:       Dict					):






								"""simple docstring"""


								A__       =    ViTHybridModelTester(self					)
								A__       =    ConfigTester(self				,					config_class=_snake_case				,					has_text_modality=_snake_case				,					hidden_size=37					)



					def     _a							(     self					:       int					):






								"""simple docstring"""


								self.config_tester.run_common_tests()



					@unittest.skip(reason='ViT does not use inputs_embeds'					)
					def     _a							(     self					:       int					):






								"""simple docstring"""


								pass



					def     _a							(     self					:       int					):






								"""simple docstring"""


								A__     , A__       =    self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
											A__       =    model_class(_snake_case					)
											self.assertIsInstance(model.get_input_embeddings()				,					(nn.Module)					)
											A__       =    model.get_output_embeddings()
											self.assertTrue(x is None or isinstance(_snake_case				,					nn.Linear					)					)



					def     _a							(     self					:       List[str]					):






								"""simple docstring"""


								A__     , A__       =    self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
											A__       =    model_class(_snake_case					)
											A__       =    inspect.signature(model.forward					)
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											A__       =    [*signature.parameters.keys()]

											A__       =    ['pixel_values']
											self.assertListEqual(arg_names[:1]				,					_snake_case					)



					def     _a							(     self					:       Any					):






								"""simple docstring"""


								A__       =    self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*_snake_case					)



					def     _a							(     self					:       str					):






								"""simple docstring"""


								A__       =    self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_image_classification(*_snake_case					)



					def     _a							(     self					:       Any					):






								"""simple docstring"""


								A__     , A__       =    self.model_tester.prepare_config_and_inputs_for_common()

								A__       =    _config_zero_init(_snake_case					)
								for model_class in self.all_model_classes:
											A__       =    model_class(config=_snake_case					)
											# Skip the check for the backbone
											for name, module in model.named_modules():
														if module.__class__.__name__ == "ViTHybridPatchEmbeddings":
																	A__       =    [F'''{name}.{key}''' for key in module.state_dict().keys()]
																	break

											for name, param in model.named_parameters():
														if param.requires_grad:
																	if name in backbone_params:
																				continue
																	self.assertIn(
																	    ((param.data.mean() * 1E9).round() / 1E9).item()				,					[0.0, 1.0]				,					msg=F'''Parameter {name} of model {model_class} seems not properly initialized'''				,					)



					@slow
					def     _a							(     self					:       int					):






								"""simple docstring"""


								for model_name in VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											A__       =    ViTHybridModel.from_pretrained(_snake_case					)
											self.assertIsNotNone(_snake_case					)







def A						(       )						->      Union[str, Any]:
			A__       =    Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'					)
			return image






@require_torch
@require_vision
class      __lowerCAmelCase     (		unittest.TestCase      ):
					"""simple docstring"""






					@cached_property
					def     _a							(     self					:       Tuple					):






								"""simple docstring"""


								return (
								    ViTHybridImageProcessor.from_pretrained(VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST[0]					)
								    if is_vision_available()
								    else None
								)



					@slow
					def     _a							(     self					:       Optional[Any]					):






								"""simple docstring"""


								A__       =    ViTHybridForImageClassification.from_pretrained(VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST[0]					).to(
								    _snake_case					)

								A__       =    self.default_image_processor
								A__       =    prepare_img()
								A__       =    image_processor(images=_snake_case				,					return_tensors='pt'					).to(_snake_case					)

								# forward pass
								with torch.no_grad():
											A__       =    model(**_snake_case					)

								# verify the logits
								A__       =    torch.Size((1, 10_00)					)
								self.assertEqual(outputs.logits.shape				,					_snake_case					)

								A__       =    torch.tensor([-1.9090, -0.4993, -0.2389]					).to(_snake_case					)

								self.assertTrue(torch.allclose(outputs.logits[0, :3]				,					_snake_case				,					atol=1E-4					)					)



					@slow
					@require_accelerate
					def     _a							(     self					:       List[Any]					):






								"""simple docstring"""


								A__       =    ViTHybridImageProcessor.from_pretrained('google/vit-hybrid-base-bit-384'					)
								A__       =    ViTHybridForImageClassification.from_pretrained('google/vit-hybrid-base-bit-384'				,					device_map='auto'					)

								A__       =    prepare_img()

								A__       =    image_processor(images=_snake_case				,					return_tensors='pt'					)
								A__       =    model(**_snake_case					)
								A__       =    outputs.logits
								# model predicts one of the 1000 ImageNet classes
								A__       =    logits.argmax(-1					).item()

								self.assertTrue(model.config.idalabel[predicted_class_idx]				,					'tabby, tabby cat'					)




