
'''simple docstring'''



from __future__ import annotations

import inspect
import unittest

from transformers import ViTConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import TFViTForImageClassification, TFViTModel


if is_vision_available():
		from PIL import Image

		from transformers import ViTImageProcessor





class _UpperCamelCase		:





				'''simple docstring'''





				def __init__(							self  :  Optional[int]  ,       _lowerCAmelCase  :  Dict  ,       _lowerCAmelCase  :  Dict=1_3  ,       _lowerCAmelCase  :  List[str]=3_0  ,       _lowerCAmelCase  :  List[str]=2  ,       _lowerCAmelCase  :  List[Any]=3  ,       _lowerCAmelCase  :  Union[str, Any]=True  ,       _lowerCAmelCase  :  Optional[int]=True  ,       _lowerCAmelCase  :  List[str]=3_2  ,       _lowerCAmelCase  :  Dict=2  ,       _lowerCAmelCase  :  List[str]=4  ,       _lowerCAmelCase  :  List[str]=3_7  ,       _lowerCAmelCase  :  Optional[Any]="gelu"  ,       _lowerCAmelCase  :  Union[str, Any]=0.1  ,       _lowerCAmelCase  :  Union[str, Any]=0.1  ,       _lowerCAmelCase  :  Tuple=1_0  ,       _lowerCAmelCase  :  Optional[Any]=0.02  ,       _lowerCAmelCase  :  Optional[int]=3  ,       _lowerCAmelCase  :  Any=None  ,       ):







											'''simple docstring'''

											__lowercase       	=parent
											__lowercase       	=batch_size
											__lowercase       	=image_size
											__lowercase       	=patch_size
											__lowercase       	=num_channels
											__lowercase       	=is_training
											__lowercase       	=use_labels
											__lowercase       	=hidden_size
											__lowercase       	=num_hidden_layers
											__lowercase       	=num_attention_heads
											__lowercase       	=intermediate_size
											__lowercase       	=hidden_act
											__lowercase       	=hidden_dropout_prob
											__lowercase       	=attention_probs_dropout_prob
											__lowercase       	=type_sequence_label_size
											__lowercase       	=initializer_range
											__lowercase       	=scope

											# in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
											__lowercase       	=(image_size // patch_size) ** 2
											__lowercase       	=num_patches + 1

				def 				__lowerCamelCase     (							self  :  str):







											'''simple docstring'''

											__lowercase       	=floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

											__lowercase       	=None
											if self.use_labels:
																		__lowercase       	=ids_tensor([self.batch_size]  ,       self.type_sequence_label_size)

											__lowercase       	=self.get_config()

											return config, pixel_values, labels

				def 				__lowerCamelCase     (							self  :  Optional[Any]):







											'''simple docstring'''

											return ViTConfig(
											    image_size=self.image_size  ,       patch_size=self.patch_size  ,       num_channels=self.num_channels  ,       hidden_size=self.hidden_size  ,       num_hidden_layers=self.num_hidden_layers  ,       num_attention_heads=self.num_attention_heads  ,       intermediate_size=self.intermediate_size  ,       hidden_act=self.hidden_act  ,       hidden_dropout_prob=self.hidden_dropout_prob  ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,       is_decoder=lowerCamelCase__  ,       initializer_range=self.initializer_range  ,       )

				def 				__lowerCamelCase     (							self  :  Any  ,       _lowerCAmelCase  :  Optional[Any]  ,       _lowerCAmelCase  :  List[Any]  ,       _lowerCAmelCase  :  str):







											'''simple docstring'''

											__lowercase       	=TFViTModel(config=lowerCamelCase__)
											__lowercase       	=model(lowerCamelCase__  ,       training=lowerCamelCase__)
											self.parent.assertEqual(result.last_hidden_state.shape  ,       (self.batch_size, self.seq_length, self.hidden_size))

											# Test with an image with different size than the one specified in config.
											__lowercase       	=self.image_size // 2
											__lowercase       	=pixel_values[:, :, :image_size, :image_size]
											__lowercase       	=model(lowerCamelCase__  ,       interpolate_pos_encoding=lowerCamelCase__  ,       training=lowerCamelCase__)
											__lowercase       	=(image_size // self.patch_size) ** 2 + 1
											self.parent.assertEqual(result.last_hidden_state.shape  ,       (self.batch_size, seq_length, self.hidden_size))

				def 				__lowerCamelCase     (							self  :  Tuple  ,       _lowerCAmelCase  :  Optional[Any]  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  Tuple):







											'''simple docstring'''

											__lowercase       	=self.type_sequence_label_size
											__lowercase       	=TFViTForImageClassification(lowerCamelCase__)
											__lowercase       	=model(lowerCamelCase__  ,       labels=lowerCamelCase__  ,       training=lowerCamelCase__)
											self.parent.assertEqual(result.logits.shape  ,       (self.batch_size, self.type_sequence_label_size))

											# Test with an image with different size than the one specified in config.
											__lowercase       	=self.image_size // 2
											__lowercase       	=pixel_values[:, :, :image_size, :image_size]
											__lowercase       	=model(lowerCamelCase__  ,       interpolate_pos_encoding=lowerCamelCase__  ,       training=lowerCamelCase__)
											self.parent.assertEqual(result.logits.shape  ,       (self.batch_size, self.type_sequence_label_size))

											# test greyscale images
											__lowercase       	=1
											__lowercase       	=TFViTForImageClassification(lowerCamelCase__)
											__lowercase       	=floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
											__lowercase       	=model(lowerCamelCase__)
											self.parent.assertEqual(result.logits.shape  ,       (self.batch_size, self.type_sequence_label_size))

				def 				__lowerCamelCase     (							self  :  Tuple):







											'''simple docstring'''

											__lowercase       	=self.prepare_config_and_inputs()
											__lowercase       	=config_and_inputs
											__lowercase       	={'''pixel_values''': pixel_values}
											return config, inputs_dict





@require_tf
class _UpperCamelCase		(     __lowerCAmelCase      ,       __lowerCAmelCase      ,       unittest.TestCase		):





				'''simple docstring'''





				lowerCAmelCase__					    =    (TFViTModel, TFViTForImageClassification) if is_tf_available() else ()
				lowerCAmelCase__					    =    (
				    {"feature-extraction": TFViTModel, "image-classification": TFViTForImageClassification}
				    if is_tf_available()
				    else {}
				)

				lowerCAmelCase__					    =    False
				lowerCAmelCase__					    =    False
				lowerCAmelCase__					    =    False

				def 				__lowerCamelCase     (							self  :  Dict):







											'''simple docstring'''

											__lowercase       	=TFViTModelTester(self)
											__lowercase       	=ConfigTester(self  ,       config_class=lowerCamelCase__  ,       has_text_modality=lowerCamelCase__  ,       hidden_size=3_7)

				def 				__lowerCamelCase     (							self  :  str):







											'''simple docstring'''

											self.config_tester.run_common_tests()

				@unittest.skip(reason='ViT does not use inputs_embeds')
				def 				__lowerCamelCase     (							self  :  List[Any]):







											'''simple docstring'''

											pass

				@unittest.skip(reason='ViT does not use inputs_embeds')
				def 				__lowerCamelCase     (							self  :  Dict):







											'''simple docstring'''

											pass

				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											__lowercase       	=self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																		__lowercase       	=model_class(lowerCamelCase__)
																		self.assertIsInstance(model.get_input_embeddings()  ,       (tf.keras.layers.Layer))
																		__lowercase       	=model.get_output_embeddings()
																		self.assertTrue(x is None or isinstance(lowerCamelCase__  ,       tf.keras.layers.Layer))

				def 				__lowerCamelCase     (							self  :  str):







											'''simple docstring'''

											__lowercase       	=self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																		__lowercase       	=model_class(lowerCamelCase__)
																		__lowercase       	=inspect.signature(model.call)
																		# signature.parameters is an OrderedDict => so arg_names order is deterministic
																		__lowercase       	=[*signature.parameters.keys()]

																		__lowercase       	=['''pixel_values''']
																		self.assertListEqual(arg_names[:1]  ,       lowerCamelCase__)

				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											__lowercase       	=self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*lowerCamelCase__)

				def 				__lowerCamelCase     (							self  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_image_classification(*lowerCamelCase__)

				@slow
				def 				__lowerCamelCase     (							self  :  int):







											'''simple docstring'''

											__lowercase       	=TFViTModel.from_pretrained('google/vit-base-patch16-224')
											self.assertIsNotNone(lowerCamelCase__)



def   _A       (   ):



							"""simple docstring"""



							__lowercase       	=Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'					)
							return image





@require_tf
@require_vision
class _UpperCamelCase		(     unittest.TestCase		):





				'''simple docstring'''





				@cached_property
				def 				__lowerCamelCase     (							self  :  Optional[Any]):







											'''simple docstring'''

											return ViTImageProcessor.from_pretrained('google/vit-base-patch16-224') if is_vision_available() else None

				@slow
				def 				__lowerCamelCase     (							self  :  int):







											'''simple docstring'''

											__lowercase       	=TFViTForImageClassification.from_pretrained('google/vit-base-patch16-224')

											__lowercase       	=self.default_image_processor
											__lowercase       	=prepare_img()
											__lowercase       	=image_processor(images=lowerCamelCase__  ,       return_tensors='tf')

											# forward pass
											__lowercase       	=model(**lowerCamelCase__)

											# verify the logits
											__lowercase       	=tf.TensorShape((1, 1_0_0_0))
											self.assertEqual(outputs.logits.shape  ,       lowerCamelCase__)

											__lowercase       	=tf.constant([-0.2744, 0.8215, -0.0836])

											tf.debugging.assert_near(outputs.logits[0, :3]  ,       lowerCamelCase__  ,       atol=1e-4)








'''simple docstring'''



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


lowerCamelCase	 =       logging.get_logger(__name__)

lowerCamelCase	 =       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase	 =       {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase	 =       {
    """Salesforce/codegen-350M-mono""": 2048,
}





class _UpperCamelCase		(     A		):





				'''simple docstring'''





				lowerCAmelCase__					    =    VOCAB_FILES_NAMES
				lowerCAmelCase__					    =    PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__					    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__					    =    ["""input_ids""", """attention_mask"""]
				lowerCAmelCase__					    =    CodeGenTokenizer

				def __init__(							self  :  int  ,       _lowerCAmelCase  :  str=None  ,       _lowerCAmelCase  :  List[Any]=None  ,       _lowerCAmelCase  :  Optional[Any]=None  ,       _lowerCAmelCase  :  List[str]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[int]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[Any]="<|endoftext|>"  ,       _lowerCAmelCase  :  Optional[int]=False  ,       **_lowerCAmelCase  :  Union[str, Any]  ,       ):







											'''simple docstring'''

											super().__init__(
											    _lowerCAmelCase  ,       _lowerCAmelCase  ,       tokenizer_file=_lowerCAmelCase  ,       unk_token=_lowerCAmelCase  ,       bos_token=_lowerCAmelCase  ,       eos_token=_lowerCAmelCase  ,       add_prefix_space=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											if kwargs.pop('add_bos_token'  ,       _lowerCAmelCase):
																		__lowercase       	=kwargs.pop('name_or_path'  ,       '')
																		raise ValueError(
																		    'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
																		    'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
																		    f"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
																		    f"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
																		    'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
																		    ' so that the fast tokenizer works correctly.')

											__lowercase       	=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
											if pre_tok_state.get('add_prefix_space'  ,       _lowerCAmelCase) != add_prefix_space:
																		__lowercase       	=getattr(_lowerCAmelCase  ,       pre_tok_state.pop('type'))
																		__lowercase       	=add_prefix_space
																		__lowercase       	=pre_tok_class(**_lowerCAmelCase)

											__lowercase       	=add_prefix_space

				def 				__lowerCamelCase     (							self  :  Any  ,       *_lowerCAmelCase  :  Tuple  ,       **_lowerCAmelCase  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)
											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  List[str]  ,       *_lowerCAmelCase  :  List[Any]  ,       **_lowerCAmelCase  :  List[str]):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)

											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  Optional[str] = None):







											'''simple docstring'''

											__lowercase       	=self._tokenizer.model.save(_lowerCAmelCase  ,       name=_lowerCAmelCase)
											return tuple(_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]  ,       _lowerCAmelCase  :  bool = False  ,       _lowerCAmelCase  :  bool = None  ,       _lowerCAmelCase  :  Optional[List[str]] = None  ,       **_lowerCAmelCase  :  Any  ,       ):







											'''simple docstring'''

											__lowercase       	=super().decode(
											    token_ids=_lowerCAmelCase  ,       skip_special_tokens=_lowerCAmelCase  ,       clean_up_tokenization_spaces=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											if truncate_before_pattern is not None and len(_lowerCAmelCase) > 0:
																		__lowercase       	=self.truncate(_lowerCAmelCase  ,       _lowerCAmelCase)

											return decoded_text

				def 				__lowerCamelCase     (							self  :  Any  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  Optional[int]):







											'''simple docstring'''

											def find_re(_lowerCAmelCase  :  List[Any]  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  str):
																		__lowercase       	=pattern.search(_lowerCAmelCase  ,       _lowerCAmelCase)
																		return m.start() if m else -1

											__lowercase       	=[re.compile(_lowerCAmelCase  ,       re.MULTILINE) for pattern in truncate_before_pattern]

											__lowercase       	=list(re.finditer('^print'  ,       _lowerCAmelCase  ,       re.MULTILINE))

											if len(_lowerCAmelCase) > 1:
																		__lowercase       	=completion[: prints[1].start()]

											__lowercase       	=list(re.finditer('^def'  ,       _lowerCAmelCase  ,       re.MULTILINE))

											if len(_lowerCAmelCase) > 1:
																		__lowercase       	=completion[: defs[1].start()]

											__lowercase       	=0

											__lowercase       	=[
											    pos for pos in [find_re(_lowerCAmelCase  ,       _lowerCAmelCase  ,       _lowerCAmelCase) for terminal in terminals] if pos != -1
											]

											if len(_lowerCAmelCase) > 0:
																		return completion[: min(_lowerCAmelCase)]
											else:
																		return completion



