





'''simple docstring'''







import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase       						=			logging.get_logger(__name__)

_lowercase       						=			'▁'

_lowercase       						=			{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

_lowercase       						=			{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

_lowercase       						=			{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

_lowercase       						=			{
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

_lowercase       						=			{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}
class 			_lowercase						(       __a					):
					_UpperCAmelCase										=					['''input_ids''']

					_UpperCAmelCase										=					VOCAB_FILES_NAMES
					_UpperCAmelCase										=					PRETRAINED_INIT_CONFIGURATION
					_UpperCAmelCase										=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_UpperCAmelCase										=					PRETRAINED_VOCAB_FILES_MAP
					_UpperCAmelCase										=					RESOURCE_FILES_NAMES







					def __init__( self    ,   A__    ,   A__=None    ,   A__=False    ,   A__="utf8"    ,   A__="[UNK]"    ,   A__="[SEP]"    ,   A__="[PAD]"    ,   A__="[CLS]"    ,   A__="[MASK]"    ,   A__ = None    ,   **A__    ,   ) ->	str:
						snake_case				=			{} if sp_model_kwargs is None else sp_model_kwargs
						super().__init__(
						    do_lower_case=__lowerCAmelCase    ,   unk_token=__lowerCAmelCase    ,   sep_token=__lowerCAmelCase    ,   pad_token=__lowerCAmelCase    ,   cls_token=__lowerCAmelCase    ,   mask_token=__lowerCAmelCase    ,   vocab_file=__lowerCAmelCase    ,   encoding=__lowerCAmelCase    ,   sp_model_kwargs=self.sp_model_kwargs    ,   **__lowerCAmelCase    ,   )
						snake_case				=			do_lower_case
						snake_case				=			sentencepiece_model_ckpt
						snake_case				=			spm.SentencePieceProcessor(**self.sp_model_kwargs    )
						self.sp_model.Load(__lowerCAmelCase    )

						# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
						if vocab_file is not None:
							snake_case				=			self.load_vocab(filepath=__lowerCAmelCase    )
						else:
							snake_case				=			{self.sp_model.id_to_piece(__lowerCAmelCase    ): id for id in range(self.sp_model.get_piece_size()    )}
						snake_case				=			{v: k for k, v in self.vocab.items()}







					def     UpperCamelCase			( self    ,   A__    ) ->	List[str]:







						if text is None:
							return None

						snake_case				=			self.tokenize(__lowerCAmelCase    )
						snake_case				,		snake_case				=			'''''', []

						for i, ch in enumerate(__lowerCAmelCase    ):
							if ch in self.SP_CHAR_MAPPING:
								snake_case				=			self.SP_CHAR_MAPPING.get(__lowerCAmelCase    )
							else:
								snake_case				=			unicodedata.normalize('''NFKC'''    ,   __lowerCAmelCase    )
							if self.is_whitespace(__lowerCAmelCase    ):
								continue
							normalized_text += ch
							char_mapping.extend([i] * len(__lowerCAmelCase    )    )

						snake_case				,		snake_case				,		snake_case				=			normalized_text, [], 0

						if self.do_lower_case:
							snake_case				=			text.lower()

						for token in split_tokens:
							if token[:1] == "▁":
								snake_case				=			token[1:]
							snake_case				=			text[offset:].index(__lowerCAmelCase    ) + offset
							snake_case				=			start + len(__lowerCAmelCase    )

							token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)    )
							snake_case				=			end
						return token_mapping







					@property
					def     UpperCamelCase			( self    ) ->	Any:
						return len(self.vocab    )







					def     UpperCamelCase			( self    ) ->	List[str]:
						return dict(self.vocab    ,   **self.added_tokens_encoder    )







					def __getstate__( self    ) ->	List[Any]:
						snake_case				=			self.__dict__.copy()
						snake_case				=			None
						return state







					def __setstate__( self    ,   A__    ) ->	Optional[int]:
						snake_case				=			d

						# for backward compatibility
						if not hasattr(self    ,   '''sp_model_kwargs'''    ):
							snake_case				=			{}

						snake_case				=			spm.SentencePieceProcessor(**self.sp_model_kwargs    )
						self.sp_model.Load(self.sentencepiece_model_ckpt    )







					def     UpperCamelCase			( self    ,   A__    ) ->	List[str]:
						return "".join((self.SP_CHAR_MAPPING.get(__lowerCAmelCase    ,   __lowerCAmelCase    ) for c in text)    )







					def     UpperCamelCase			( self    ,   A__    ,   A__=False    ,   A__=64    ,   A__=0.1    ) ->	List[Any]:







						if self.sp_model_kwargs.get('''enable_sampling'''    ) is True:
							snake_case				=			True
						if self.sp_model_kwargs.get('''alpha'''    ) is not None:
							snake_case				=			self.sp_model_kwargs.get('''alpha'''    )
						if self.sp_model_kwargs.get('''nbest_size'''    ) is not None:
							snake_case				=			self.sp_model_kwargs.get('''nbest_size'''    )

						if not enable_sampling:
							snake_case				=			self.sp_model.EncodeAsPieces(__lowerCAmelCase    )
						else:
							snake_case				=			self.sp_model.SampleEncodeAsPieces(__lowerCAmelCase    ,   __lowerCAmelCase    ,   __lowerCAmelCase    )
						snake_case				=			[]
						for pi, piece in enumerate(__lowerCAmelCase    ):
							if piece == SPIECE_UNDERLINE:
								if not pieces[pi + 1].startswith(__lowerCAmelCase    ) and pi != 0:
									new_pieces.append(__lowerCAmelCase    )
									continue
								else:
									continue
							snake_case				=			0
							for i, chunk in enumerate(__lowerCAmelCase    ):
								if chunk == SPIECE_UNDERLINE:
									continue
								if self.is_ch_char(__lowerCAmelCase    ) or self.is_punct(__lowerCAmelCase    ):
									if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
										new_pieces.append(piece[lst_i:i]    )
									new_pieces.append(__lowerCAmelCase    )
									snake_case				=			i + 1
								elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
									if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
										new_pieces.append(piece[lst_i:i]    )
									snake_case				=			i
								elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
									if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
										new_pieces.append(piece[lst_i:i]    )
									snake_case				=			i
							if len(__lowerCAmelCase    ) > lst_i:
								new_pieces.append(piece[lst_i:]    )
						return new_pieces







					def     UpperCamelCase			( self    ,   A__    ) ->	List[str]:
						snake_case				=			''''''.join(__lowerCAmelCase    ).replace(__lowerCAmelCase    ,   ''' '''    ).strip()
						return out_string







					def     UpperCamelCase			( self    ,   A__    ) ->	Dict:
						snake_case				=			self.convert_ids_to_tokens(__lowerCAmelCase    )
						snake_case				=			''''''.join(__lowerCAmelCase    ).replace(__lowerCAmelCase    ,   ''' '''    ).strip()
						return out_string







					def     UpperCamelCase			( self    ,   A__    ) ->	Any:
						return self.vocab.get(__lowerCAmelCase    ,   self.vocab.get(self.unk_token    )    )







					def     UpperCamelCase			( self    ,   A__    ) ->	Dict:
						return self.reverse_vocab.get(__lowerCAmelCase    ,   self.unk_token    )







					def     UpperCamelCase			( self    ,   A__    ,   A__=None    ) ->	Optional[Any]:







						if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						snake_case				=			[self.cls_token_id]
						snake_case				=			[self.sep_token_id]
						return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep







					def     UpperCamelCase			( self    ,   A__    ,   A__=None    ) ->	Tuple:







						if offset_mapping_a is None:
							return [(0, 0)] + offset_mapping_a + [(0, 0)]

						return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]







					def     UpperCamelCase			( self    ,   A__    ,   A__=None    ,   A__=False    ) ->	Tuple:







						if already_has_special_tokens:
							if token_ids_a is not None:
								raise ValueError(
								    '''You should not supply a second sequence if the provided sequence of '''
								    '''ids is already formatted with special tokens for the model.'''    )
							return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

						if token_ids_a is not None:
							return [1] + ([0] * len(__lowerCAmelCase    )) + [1, 1] + ([0] * len(__lowerCAmelCase    )) + [1]
						return [1] + ([0] * len(__lowerCAmelCase    )) + [1]







					def     UpperCamelCase			( self    ,   A__    ,   A__ = None    ) ->	Dict:







						if token_ids_a is None:
							# [CLS] X [SEP]
							return (len(__lowerCAmelCase    ) + 2) * [0]

						# [CLS] A [SEP] [SEP] B [SEP]
						return [0] * (len(__lowerCAmelCase    ) + 1) + [1] * (len(__lowerCAmelCase    ) + 3)







					def     UpperCamelCase			( self    ,   A__    ) ->	Optional[int]:







						if "\u4e00" <= char <= "\u9fff":
							return True
						return False







					def     UpperCamelCase			( self    ,   A__    ) ->	Dict:







						if ("a" <= char <= "z") or ("A" <= char <= "Z"):
							return True
						return False







					def     UpperCamelCase			( self    ,   A__    ) ->	List[str]:







						if char in ",;:.?!~，；：。？！《》【】":
							return True
						return False







					def     UpperCamelCase			( self    ,   A__    ) ->	Union[str, Any]:







						if char == " " or char == "\t" or char == "\n" or char == "\r":
							return True
						if len(__lowerCAmelCase    ) == 1:
							snake_case				=			unicodedata.category(__lowerCAmelCase    )
							if cat == "Zs":
								return True
						return False







					def     UpperCamelCase			( self    ,   A__    ) ->	List[Any]:
						snake_case				=			{}
						with io.open(__lowerCAmelCase    ,   '''r'''    ,   encoding='''utf-8'''    ) as f:
							for index, line in enumerate(__lowerCAmelCase    ):
								snake_case				=			line.rstrip('''\n'''    )
								snake_case				=			int(__lowerCAmelCase    )

						return token_to_idx







					def     UpperCamelCase			( self    ,   A__    ,   A__ = None    ) ->	Union[str, Any]:
						snake_case				=			0
						if os.path.isdir(__lowerCAmelCase    ):
							snake_case				=			os.path.join(
							    __lowerCAmelCase    ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
						else:
							snake_case				=			(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
						with open(__lowerCAmelCase    ,   '''w'''    ,   encoding='''utf-8'''    ) as writer:
							for token, token_index in sorted(self.vocab.items()    ,   key=lambda A__    : kv[1]    ):
								if index != token_index:
									logger.warning(
									    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
									    ''' Please check that the vocabulary is not corrupted!'''    )
									snake_case				=			token_index
								writer.write(token + '''\n'''    )
								index += 1

						snake_case				=			os.path.join(__lowerCAmelCase    ,   '''sentencepiece.bpe.model'''    )
						with open(__lowerCAmelCase    ,   '''wb'''    ) as fi:
							snake_case				=			self.sp_model.serialized_model_proto()
							fi.write(__lowerCAmelCase    )

						return (vocab_file,)








'''simple docstring'''







from __future__ import annotations

import inspect
import unittest

from transformers import ViTConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers import TFViTForImageClassification, TFViTModel


if is_vision_available():
   from PIL import Image

   from transformers import ViTImageProcessor
class 			_lowercase						:







     def __init__( self    ,   A__    ,   A__=13    ,   A__=30    ,   A__=2    ,   A__=3    ,   A__=True    ,   A__=True    ,   A__=32    ,   A__=2    ,   A__=4    ,   A__=37    ,   A__="gelu"    ,   A__=0.1    ,   A__=0.1    ,   A__=10    ,   A__=0.0_2    ,   A__=3    ,   A__=None    ,   ) ->	List[Any]:
      snake_case				=			parent
      snake_case				=			batch_size
      snake_case				=			image_size
      snake_case				=			patch_size
      snake_case				=			num_channels
      snake_case				=			is_training
      snake_case				=			use_labels
      snake_case				=			hidden_size
      snake_case				=			num_hidden_layers
      snake_case				=			num_attention_heads
      snake_case				=			intermediate_size
      snake_case				=			hidden_act
      snake_case				=			hidden_dropout_prob
      snake_case				=			attention_probs_dropout_prob
      snake_case				=			type_sequence_label_size
      snake_case				=			initializer_range
      snake_case				=			scope

      # in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
      snake_case				=			(image_size // patch_size) ** 2
      snake_case				=			num_patches + 1







     def     UpperCamelCase			( self    ) ->	int:
      snake_case				=			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]    )

      snake_case				=			None
      if self.use_labels:
       snake_case				=			ids_tensor([self.batch_size]    ,   self.type_sequence_label_size    )

      snake_case				=			self.get_config()

      return config, pixel_values, labels







     def     UpperCamelCase			( self    ) ->	int:
      return ViTConfig(
          image_size=self.image_size    ,   patch_size=self.patch_size    ,   num_channels=self.num_channels    ,   hidden_size=self.hidden_size    ,   num_hidden_layers=self.num_hidden_layers    ,   num_attention_heads=self.num_attention_heads    ,   intermediate_size=self.intermediate_size    ,   hidden_act=self.hidden_act    ,   hidden_dropout_prob=self.hidden_dropout_prob    ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,   is_decoder=A__    ,   initializer_range=self.initializer_range    ,   )







     def     UpperCamelCase			( self    ,   A__    ,   A__    ,   A__    ) ->	Union[str, Any]:
      snake_case				=			TFViTModel(config=A__    )
      snake_case				=			model(A__    ,   training=A__    )
      self.parent.assertEqual(result.last_hidden_state.shape    ,   (self.batch_size, self.seq_length, self.hidden_size)    )

      # Test with an image with different size than the one specified in config.
      snake_case				=			self.image_size // 2
      snake_case				=			pixel_values[:, :, :image_size, :image_size]
      snake_case				=			model(A__    ,   interpolate_pos_encoding=A__    ,   training=A__    )
      snake_case				=			(image_size // self.patch_size) ** 2 + 1
      self.parent.assertEqual(result.last_hidden_state.shape    ,   (self.batch_size, seq_length, self.hidden_size)    )







     def     UpperCamelCase			( self    ,   A__    ,   A__    ,   A__    ) ->	Optional[int]:
      snake_case				=			self.type_sequence_label_size
      snake_case				=			TFViTForImageClassification(A__    )
      snake_case				=			model(A__    ,   labels=A__    ,   training=A__    )
      self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.type_sequence_label_size)    )

      # Test with an image with different size than the one specified in config.
      snake_case				=			self.image_size // 2
      snake_case				=			pixel_values[:, :, :image_size, :image_size]
      snake_case				=			model(A__    ,   interpolate_pos_encoding=A__    ,   training=A__    )
      self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.type_sequence_label_size)    )

      # test greyscale images
      snake_case				=			1
      snake_case				=			TFViTForImageClassification(A__    )
      snake_case				=			floats_tensor([self.batch_size, 1, self.image_size, self.image_size]    )
      snake_case				=			model(A__    )
      self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.type_sequence_label_size)    )







     def     UpperCamelCase			( self    ) ->	Union[str, Any]:
      snake_case				=			self.prepare_config_and_inputs()
      snake_case				,		snake_case				,		snake_case				=			config_and_inputs
      snake_case				=			{'''pixel_values''': pixel_values}
      return config, inputs_dict




@require_tf
class 			_lowercase						(       __a				,		__a				,		unittest.TestCase					):
     _UpperCAmelCase										=					(TFViTModel, TFViTForImageClassification) if is_tf_available() else ()
     _UpperCAmelCase										=					(
         {'''feature-extraction''': TFViTModel, '''image-classification''': TFViTForImageClassification}
         if is_tf_available()
         else {}
     )

     _UpperCAmelCase										=					False
     _UpperCAmelCase										=					False
     _UpperCAmelCase										=					False







     def     UpperCamelCase			( self    ) ->	List[Any]:
      snake_case				=			TFViTModelTester(self    )
      snake_case				=			ConfigTester(self    ,   config_class=A__    ,   has_text_modality=A__    ,   hidden_size=37    )







     def     UpperCamelCase			( self    ) ->	int:
      self.config_tester.run_common_tests()







     @unittest.skip(reason='''ViT does not use inputs_embeds'''    )
     def     UpperCamelCase			( self    ) ->	int:
      pass







     @unittest.skip(reason='''ViT does not use inputs_embeds'''    )
     def     UpperCamelCase			( self    ) ->	str:
      pass







     def     UpperCamelCase			( self    ) ->	Union[str, Any]:
      snake_case				,		snake_case				=			self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       snake_case				=			model_class(A__    )
       self.assertIsInstance(model.get_input_embeddings()    ,   (tf.keras.layers.Layer)    )
       snake_case				=			model.get_output_embeddings()
       self.assertTrue(x is None or isinstance(A__    ,   tf.keras.layers.Layer    )    )







     def     UpperCamelCase			( self    ) ->	List[Any]:
      snake_case				,		snake_case				=			self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       snake_case				=			model_class(A__    )
       snake_case				=			inspect.signature(model.call    )
       # signature.parameters is an OrderedDict => so arg_names order is deterministic
       snake_case				=			[*signature.parameters.keys()]

       snake_case				=			['''pixel_values''']
       self.assertListEqual(arg_names[:1]    ,   A__    )







     def     UpperCamelCase			( self    ) ->	Union[str, Any]:
      snake_case				=			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*A__    )







     def     UpperCamelCase			( self    ) ->	Optional[Any]:
      snake_case				=			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_image_classification(*A__    )







     @slow
     def     UpperCamelCase			( self    ) ->	Any:
      snake_case				=			TFViTModel.from_pretrained('''google/vit-base-patch16-224'''    )
      self.assertIsNotNone(A__    )




def    __UpperCamelCase	(	) ->Any:
 snake_case				=			Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''     )
 return image






@require_tf
@require_vision
class 			_lowercase						(       unittest.TestCase					):







     @cached_property
     def     UpperCamelCase			( self    ) ->	Optional[int]:
      return ViTImageProcessor.from_pretrained('''google/vit-base-patch16-224'''    ) if is_vision_available() else None







     @slow
     def     UpperCamelCase			( self    ) ->	Dict:
      snake_case				=			TFViTForImageClassification.from_pretrained('''google/vit-base-patch16-224'''    )

      snake_case				=			self.default_image_processor
      snake_case				=			prepare_img()
      snake_case				=			image_processor(images=A__    ,   return_tensors='''tf'''    )

      # forward pass
      snake_case				=			model(**A__    )

      # verify the logits
      snake_case				=			tf.TensorShape((1, 10_00)    )
      self.assertEqual(outputs.logits.shape    ,   A__    )

      snake_case				=			tf.constant([-0.2_7_4_4, 0.8_2_1_5, -0.0_8_3_6]    )

      tf.debugging.assert_near(outputs.logits[0, :3]    ,   A__    ,   atol=1e-4    )

