

from typing import Optional, Tuple, Union

import tensorflow as tf

from ...activations_tf import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_tf_outputs import (
    TFBaseModelOutputWithNoAttention,
    TFBaseModelOutputWithPoolingAndNoAttention,
    TFSequenceClassifierOutput,
)
from ...modeling_tf_utils import TFPreTrainedModel, TFSequenceClassificationLoss, keras_serializable, unpack_inputs
from ...tf_utils import shape_list
from ...utils import logging
from .configuration_regnet import RegNetConfig


_SCREAMING_SNAKE_CASE  :						Any         =    logging.get_logger(__name__)

# General docstring
_SCREAMING_SNAKE_CASE  :						str         =    'RegNetConfig'

# Base docstring
_SCREAMING_SNAKE_CASE  :						Tuple         =    'facebook/regnet-y-040'
_SCREAMING_SNAKE_CASE  :						Optional[int]         =    [1, 1088, 7, 7]

# Image classification docstring
_SCREAMING_SNAKE_CASE  :						Dict         =    'facebook/regnet-y-040'
_SCREAMING_SNAKE_CASE  :						Tuple         =    'tabby, tabby cat'

_SCREAMING_SNAKE_CASE  :						Optional[Any]         =    [
    'facebook/regnet-y-040',
    # See all regnet models at https://huggingface.co/models?filter=regnet
]






class  UpperCAmelCase__	( tf.keras.layers.Layer ):






  """simple docstring"""


  def __init__(							self		:	Any			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	int = 3			,     __lowerCamelCase		:	int = 1			,     __lowerCamelCase		:	int = 1			,     __lowerCamelCase		:	Optional[str] = "relu"			,     **__lowerCamelCase		:	str			,     )	->							Tuple:
       super().__init__(**__A      )
       # The padding and conv has been verified in
       # https://colab.research.google.com/gist/sayakpaul/854bc10eeaf21c9ee2119e0b9f3841a7/scratchpad.ipynb
       SCREAMING_SNAKE_CASE__					=					tf.keras.layers.ZeroPaddingaD(padding=kernel_size // 2      )
       SCREAMING_SNAKE_CASE__					=					tf.keras.layers.ConvaD(
           filters=__A			,     kernel_size=__A			,     strides=__A			,     padding='''VALID'''			,     groups=__A			,     use_bias=__A			,     name='''convolution'''			,     )
       SCREAMING_SNAKE_CASE__					=					tf.keras.layers.BatchNormalization(epsilon=1e-5			,     momentum=0.9			,     name='''normalization'''      )
       SCREAMING_SNAKE_CASE__					=					ACTaFN[activation] if activation is not None else tf.identity







  def    lowercase_	(							self		:	Any			,     __lowerCamelCase		:	Optional[int]      )	->							List[str]:
       SCREAMING_SNAKE_CASE__					=					self.convolution(self.padding(__A      )      )
       SCREAMING_SNAKE_CASE__					=					self.normalization(__A      )
       SCREAMING_SNAKE_CASE__					=					self.activation(__A      )
       return hidden_state






class  UpperCAmelCase__	( tf.keras.layers.Layer ):






  """simple docstring"""


  def __init__(							self		:	Tuple			,     __lowerCamelCase		:	RegNetConfig			,     **__lowerCamelCase		:	List[Any]      )	->							Union[str, Any]:
       super().__init__(**__A      )
       SCREAMING_SNAKE_CASE__					=					config.num_channels
       SCREAMING_SNAKE_CASE__					=					TFRegNetConvLayer(
           out_channels=config.embedding_size			,     kernel_size=3			,     stride=2			,     activation=config.hidden_act			,     name='''embedder'''			,     )







  def    lowercase_	(							self		:	Dict			,     __lowerCamelCase		:	str      )	->							str:
       SCREAMING_SNAKE_CASE__					=					shape_list(__A      )[1]
       if tf.executing_eagerly() and num_channels != self.num_channels:
            raise ValueError(
                '''Make sure that the channel dimension of the pixel values match with the one set in the configuration.'''      )

       # When running on CPU, `tf.keras.layers.Conv2D` doesn't support `NCHW` format.
       # So change the input format from `NCHW` to `NHWC`.
       # shape = (batch_size, in_height, in_width, in_channels=num_channels)
       SCREAMING_SNAKE_CASE__					=					tf.transpose(__A			,     perm=(0, 2, 3, 1)      )
       SCREAMING_SNAKE_CASE__					=					self.embedder(__A      )
       return hidden_state






class  UpperCAmelCase__	( tf.keras.layers.Layer ):






  """simple docstring"""


  def __init__(							self		:	Union[str, Any]			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	int = 2			,     **__lowerCamelCase		:	List[str]      )	->							List[str]:
       super().__init__(**__A      )
       SCREAMING_SNAKE_CASE__					=					tf.keras.layers.ConvaD(
           filters=__A			,     kernel_size=1			,     strides=__A			,     use_bias=__A			,     name='''convolution'''      )
       SCREAMING_SNAKE_CASE__					=					tf.keras.layers.BatchNormalization(epsilon=1e-5			,     momentum=0.9			,     name='''normalization'''      )







  def    lowercase_	(							self		:	str			,     __lowerCamelCase		:	tf.Tensor			,     __lowerCamelCase		:	bool = False      )	->							int:
       return self.normalization(self.convolution(__A      )			,     training=__A      )






class  UpperCAmelCase__	( tf.keras.layers.Layer ):






  """simple docstring"""


  def __init__(							self		:	Union[str, Any]			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	int			,     **__lowerCamelCase		:	Optional[Any]      )	->							List[str]:
       super().__init__(**__A      )
       SCREAMING_SNAKE_CASE__					=					tf.keras.layers.GlobalAveragePoolingaD(keepdims=__A			,     name='''pooler'''      )
       SCREAMING_SNAKE_CASE__					=					[
           tf.keras.layers.ConvaD(filters=__A			,     kernel_size=1			,     activation='''relu'''			,     name='''attention.0'''      ),
           tf.keras.layers.ConvaD(filters=__A			,     kernel_size=1			,     activation='''sigmoid'''			,     name='''attention.2'''      ),
       ]







  def    lowercase_	(							self		:	Optional[int]			,     __lowerCamelCase		:	int      )	->							List[Any]:
       # [batch_size, h, w, num_channels] -> [batch_size, 1, 1, num_channels]
       SCREAMING_SNAKE_CASE__					=					self.pooler(__A      )
       for layer_module in self.attention:
            SCREAMING_SNAKE_CASE__					=					layer_module(__A      )
       SCREAMING_SNAKE_CASE__					=					hidden_state * pooled
       return hidden_state






class  UpperCAmelCase__	( tf.keras.layers.Layer ):






  """simple docstring"""


  def __init__(							self		:	Any			,     __lowerCamelCase		:	RegNetConfig			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	int = 1			,     **__lowerCamelCase		:	str      )	->							Dict:
       super().__init__(**__A      )
       SCREAMING_SNAKE_CASE__					=					in_channels != out_channels or stride != 1
       SCREAMING_SNAKE_CASE__					=					max(1			,     out_channels // config.groups_width      )
       SCREAMING_SNAKE_CASE__					=					(
           TFRegNetShortCut(__A			,     stride=__A			,     name='''shortcut'''      )
           if should_apply_shortcut
           else tf.keras.layers.Activation('''linear'''			,     name='''shortcut'''      )
       )
       # `self.layers` instead of `self.layer` because that is a reserved argument.
       SCREAMING_SNAKE_CASE__					=					[
           TFRegNetConvLayer(__A			,     kernel_size=1			,     activation=config.hidden_act			,     name='''layer.0'''      ),
           TFRegNetConvLayer(
               __A			,     stride=__A			,     groups=__A			,     activation=config.hidden_act			,     name='''layer.1'''      ),
           TFRegNetConvLayer(__A			,     kernel_size=1			,     activation=__A			,     name='''layer.2'''      ),
       ]
       SCREAMING_SNAKE_CASE__					=					ACTaFN[config.hidden_act]







  def    lowercase_	(							self		:	Tuple			,     __lowerCamelCase		:	Optional[int]      )	->							int:
       SCREAMING_SNAKE_CASE__					=					hidden_state
       for layer_module in self.layers:
            SCREAMING_SNAKE_CASE__					=					layer_module(__A      )
       SCREAMING_SNAKE_CASE__					=					self.shortcut(__A      )
       hidden_state += residual
       SCREAMING_SNAKE_CASE__					=					self.activation(__A      )
       return hidden_state






class  UpperCAmelCase__	( tf.keras.layers.Layer ):






  """simple docstring"""


  def __init__(							self		:	Union[str, Any]			,     __lowerCamelCase		:	RegNetConfig			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	int = 1			,     **__lowerCamelCase		:	List[str]      )	->							str:
       super().__init__(**__A      )
       SCREAMING_SNAKE_CASE__					=					in_channels != out_channels or stride != 1
       SCREAMING_SNAKE_CASE__					=					max(1			,     out_channels // config.groups_width      )
       SCREAMING_SNAKE_CASE__					=					(
           TFRegNetShortCut(__A			,     stride=__A			,     name='''shortcut'''      )
           if should_apply_shortcut
           else tf.keras.layers.Activation('''linear'''			,     name='''shortcut'''      )
       )
       SCREAMING_SNAKE_CASE__					=					[
           TFRegNetConvLayer(__A			,     kernel_size=1			,     activation=config.hidden_act			,     name='''layer.0'''      ),
           TFRegNetConvLayer(
               __A			,     stride=__A			,     groups=__A			,     activation=config.hidden_act			,     name='''layer.1'''      ),
           TFRegNetSELayer(__A			,     reduced_channels=int(round(in_channels / 4      )      )			,     name='''layer.2'''      ),
           TFRegNetConvLayer(__A			,     kernel_size=1			,     activation=__A			,     name='''layer.3'''      ),
       ]
       SCREAMING_SNAKE_CASE__					=					ACTaFN[config.hidden_act]







  def    lowercase_	(							self		:	str			,     __lowerCamelCase		:	str      )	->							Optional[int]:
       SCREAMING_SNAKE_CASE__					=					hidden_state
       for layer_module in self.layers:
            SCREAMING_SNAKE_CASE__					=					layer_module(__A      )
       SCREAMING_SNAKE_CASE__					=					self.shortcut(__A      )
       hidden_state += residual
       SCREAMING_SNAKE_CASE__					=					self.activation(__A      )
       return hidden_state






class  UpperCAmelCase__	( tf.keras.layers.Layer ):






  """simple docstring"""


  def __init__(							self		:	str			,     __lowerCamelCase		:	RegNetConfig			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	int = 2			,     __lowerCamelCase		:	int = 2			,     **__lowerCamelCase		:	Dict      )	->							Dict:
       super().__init__(**__A      )

       SCREAMING_SNAKE_CASE__					=					TFRegNetXLayer if config.layer_type == "x" else TFRegNetYLayer
       SCREAMING_SNAKE_CASE__					=					[
           # downsampling is done in the first layer with stride of 2
           layer(__A			,     __A			,     __A			,     stride=__A			,     name='''layers.0'''      ),
           *[layer(__A			,     __A			,     __A			,     name=f'''layers.{i+1}'''      ) for i in range(depth - 1      )],
       ]







  def    lowercase_	(							self		:	Dict			,     __lowerCamelCase		:	Dict      )	->							str:
       for layer_module in self.layers:
            SCREAMING_SNAKE_CASE__					=					layer_module(__A      )
       return hidden_state






class  UpperCAmelCase__	( tf.keras.layers.Layer ):






  """simple docstring"""


  def __init__(							self		:	str			,     __lowerCamelCase		:	RegNetConfig			,     **__lowerCamelCase		:	Union[str, Any]      )	->							str:
       super().__init__(**__A      )
       SCREAMING_SNAKE_CASE__					=					[]
       # based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
       self.stages.append(
           TFRegNetStage(
               __A			,     config.embedding_size			,     config.hidden_sizes[0]			,     stride=2 if config.downsample_in_first_stage else 1			,     depth=config.depths[0]			,     name='''stages.0'''			,     )      )
       SCREAMING_SNAKE_CASE__					=					zip(config.hidden_sizes			,     config.hidden_sizes[1:]      )
       for i, ((in_channels, out_channels), depth) in enumerate(zip(__A			,     config.depths[1:]      )      ):
            self.stages.append(TFRegNetStage(__A			,     __A			,     __A			,     depth=__A			,     name=f'''stages.{i+1}'''      )      )







  def    lowercase_	(							self		:	Optional[int]			,     __lowerCamelCase		:	tf.Tensor			,     __lowerCamelCase		:	bool = False			,     __lowerCamelCase		:	bool = True      )	->							int:
       SCREAMING_SNAKE_CASE__					=					() if output_hidden_states else None

       for stage_module in self.stages:
            if output_hidden_states:
                 SCREAMING_SNAKE_CASE__					=					hidden_states + (hidden_state,)

            SCREAMING_SNAKE_CASE__					=					stage_module(__A      )

       if output_hidden_states:
            SCREAMING_SNAKE_CASE__					=					hidden_states + (hidden_state,)

       if not return_dict:
            return tuple(v for v in [hidden_state, hidden_states] if v is not None      )

       return TFBaseModelOutputWithNoAttention(last_hidden_state=__A			,     hidden_states=__A      )






@keras_serializable
class  UpperCAmelCase__	( tf.keras.layers.Layer ):






  """simple docstring"""


  a     =      RegNetConfig






  def __init__(							self		:	Dict			,     __lowerCamelCase		:	List[str]			,     **__lowerCamelCase		:	Union[str, Any]      )	->							Dict:
       super().__init__(**__A      )
       SCREAMING_SNAKE_CASE__					=					config
       SCREAMING_SNAKE_CASE__					=					TFRegNetEmbeddings(__A			,     name='''embedder'''      )
       SCREAMING_SNAKE_CASE__					=					TFRegNetEncoder(__A			,     name='''encoder'''      )
       SCREAMING_SNAKE_CASE__					=					tf.keras.layers.GlobalAveragePoolingaD(keepdims=__A			,     name='''pooler'''      )







  @unpack_inputs
  def    lowercase_	(							self		:	Union[str, Any]			,     __lowerCamelCase		:	tf.Tensor			,     __lowerCamelCase		:	Optional[bool] = None			,     __lowerCamelCase		:	Optional[bool] = None			,     __lowerCamelCase		:	bool = False			,     )	->							Union[str, Any]:
       SCREAMING_SNAKE_CASE__					=					(
           output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
       )
       SCREAMING_SNAKE_CASE__					=					return_dict if return_dict is not None else self.config.use_return_dict

       SCREAMING_SNAKE_CASE__					=					self.embedder(__A			,     training=__A      )

       SCREAMING_SNAKE_CASE__					=					self.encoder(
           __A			,     output_hidden_states=__A			,     return_dict=__A			,     training=__A      )

       SCREAMING_SNAKE_CASE__					=					encoder_outputs[0]
       SCREAMING_SNAKE_CASE__					=					self.pooler(__A      )

       # Change to NCHW output format have uniformity in the modules
       SCREAMING_SNAKE_CASE__					=					tf.transpose(__A			,     perm=(0, 3, 1, 2)      )
       SCREAMING_SNAKE_CASE__					=					tf.transpose(__A			,     perm=(0, 3, 1, 2)      )

       # Change the other hidden state outputs to NCHW as well
       if output_hidden_states:
            SCREAMING_SNAKE_CASE__					=					tuple([tf.transpose(__A			,     perm=(0, 3, 1, 2)      ) for h in encoder_outputs[1]]      )

       if not return_dict:
            return (last_hidden_state, pooled_output) + encoder_outputs[1:]

       return TFBaseModelOutputWithPoolingAndNoAttention(
           last_hidden_state=__A			,     pooler_output=__A			,     hidden_states=hidden_states if output_hidden_states else encoder_outputs.hidden_states			,     )






class  UpperCAmelCase__	( A__ ):






  """simple docstring"""


  a     =      RegNetConfig
  a     =      "regnet"
  a     =      "pixel_values"






  @property
  def    lowercase_	(							self		:	List[str]      )	->							Any:
       return {"pixel_values": tf.TensorSpec(shape=(None, self.config.num_channels, 224, 224)			,     dtype=tf.floataa      )}



_SCREAMING_SNAKE_CASE  :						str         =    R'\n    Parameters:\n    This model is a Tensorflow\n    [tf.keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) sub-class. Use it as a\n    regular Tensorflow Module and refer to the Tensorflow documentation for all matter related to general usage and\n    behavior.\n        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.\n'

_SCREAMING_SNAKE_CASE  :						int         =    R'\n    Args:\n        pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`ConveNextImageProcessor.__call__`] for details.\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n'






@add_start_docstrings(
    "The bare RegNet model outputting raw features without any specific head on top."		,  A__		,  )
class  UpperCAmelCase__	( A__ ):






  """simple docstring"""


  def __init__(							self		:	Optional[Any]			,     __lowerCamelCase		:	RegNetConfig			,     *__lowerCamelCase		:	int			,     **__lowerCamelCase		:	Optional[Any]      )	->							List[str]:
       super().__init__(__A			,     *__A			,     **__A      )
       SCREAMING_SNAKE_CASE__					=					TFRegNetMainLayer(__A			,     name='''regnet'''      )







  @unpack_inputs
  @add_start_docstrings_to_model_forward(__A      )
  @add_code_sample_docstrings(
      checkpoint=_CHECKPOINT_FOR_DOC			,     output_type=__A			,     config_class=_CONFIG_FOR_DOC			,     modality='''vision'''			,     expected_output=_EXPECTED_OUTPUT_SHAPE			,     )
  def    lowercase_	(							self		:	str			,     __lowerCamelCase		:	tf.Tensor			,     __lowerCamelCase		:	Optional[bool] = None			,     __lowerCamelCase		:	Optional[bool] = None			,     __lowerCamelCase		:	Optional[int]=False			,     )	->							int:
       SCREAMING_SNAKE_CASE__					=					(
           output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
       )
       SCREAMING_SNAKE_CASE__					=					return_dict if return_dict is not None else self.config.use_return_dict

       SCREAMING_SNAKE_CASE__					=					self.regnet(
           pixel_values=__A			,     output_hidden_states=__A			,     return_dict=__A			,     training=__A			,     )
       if not return_dict:
            return (outputs[0],) + outputs[1:]

       return TFBaseModelOutputWithPoolingAndNoAttention(
           last_hidden_state=outputs.last_hidden_state			,     pooler_output=outputs.pooler_output			,     hidden_states=outputs.hidden_states			,     )






@add_start_docstrings(
    "\n    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    "		,  A__		,  )
class  UpperCAmelCase__	( A__		,  A__ ):






  """simple docstring"""


  def __init__(							self		:	Union[str, Any]			,     __lowerCamelCase		:	RegNetConfig			,     *__lowerCamelCase		:	Optional[int]			,     **__lowerCamelCase		:	Tuple      )	->							Tuple:
       super().__init__(__A			,     *__A			,     **__A      )
       SCREAMING_SNAKE_CASE__					=					config.num_labels
       SCREAMING_SNAKE_CASE__					=					TFRegNetMainLayer(__A			,     name='''regnet'''      )
       # classification head
       SCREAMING_SNAKE_CASE__					=					[
           tf.keras.layers.Flatten(),
           tf.keras.layers.Dense(config.num_labels			,     name='''classifier.1'''      ) if config.num_labels > 0 else tf.identity,
       ]







  @unpack_inputs
  @add_start_docstrings_to_model_forward(__A      )
  @add_code_sample_docstrings(
      checkpoint=_IMAGE_CLASS_CHECKPOINT			,     output_type=__A			,     config_class=_CONFIG_FOR_DOC			,     expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT			,     )
  def    lowercase_	(							self		:	Optional[Any]			,     __lowerCamelCase		:	tf.Tensor = None			,     __lowerCamelCase		:	tf.Tensor = None			,     __lowerCamelCase		:	bool = None			,     __lowerCamelCase		:	bool = None			,     __lowerCamelCase		:	Union[str, Any]=False			,     )	->							List[Any]:
       SCREAMING_SNAKE_CASE__					=					(
           output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
       )
       SCREAMING_SNAKE_CASE__					=					return_dict if return_dict is not None else self.config.use_return_dict

       SCREAMING_SNAKE_CASE__					=					self.regnet(
           __A			,     output_hidden_states=__A			,     return_dict=__A			,     training=__A      )

       SCREAMING_SNAKE_CASE__					=					outputs.pooler_output if return_dict else outputs[1]

       SCREAMING_SNAKE_CASE__					=					self.classifier[0](__A      )
       SCREAMING_SNAKE_CASE__					=					self.classifier[1](__A      )

       SCREAMING_SNAKE_CASE__					=					None if labels is None else self.hf_compute_loss(labels=__A			,     logits=__A      )

       if not return_dict:
            SCREAMING_SNAKE_CASE__					=					(logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

       return TFSequenceClassifierOutput(loss=__A			,     logits=__A			,     hidden_states=outputs.hidden_states      )





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_SCREAMING_SNAKE_CASE  :						Union[str, Any]         =    logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE  :						int         =    {'''vocab_file''': '''spiece.model'''}

_SCREAMING_SNAKE_CASE  :						int         =    {
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

_SCREAMING_SNAKE_CASE  :						List[Any]         =    {
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
_SCREAMING_SNAKE_CASE  :						Optional[int]         =    0
_SCREAMING_SNAKE_CASE  :						Optional[Any]         =    1
_SCREAMING_SNAKE_CASE  :						int         =    2
_SCREAMING_SNAKE_CASE  :						str         =    3
_SCREAMING_SNAKE_CASE  :						str         =    4






class  UpperCAmelCase__	( A__ ):






  """simple docstring"""


  a     =      VOCAB_FILES_NAMES
  a     =      PRETRAINED_VOCAB_FILES_MAP
  a     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a     =      "left"






  def __init__(							self		:	Optional[int]			,     __lowerCamelCase		:	Tuple			,     __lowerCamelCase		:	List[str]=False			,     __lowerCamelCase		:	Optional[Any]=True			,     __lowerCamelCase		:	Optional[Any]=False			,     __lowerCamelCase		:	List[str]="<s>"			,     __lowerCamelCase		:	Any="</s>"			,     __lowerCamelCase		:	Tuple="<unk>"			,     __lowerCamelCase		:	List[str]="<sep>"			,     __lowerCamelCase		:	List[str]="<pad>"			,     __lowerCamelCase		:	int="<cls>"			,     __lowerCamelCase		:	Any="<mask>"			,     __lowerCamelCase		:	Dict=["<eop>", "<eod>"]			,     __lowerCamelCase		:	Optional[Dict[str, Any]] = None			,     **__lowerCamelCase		:	int			,     )	->							None:
       # Mask token behave like a normal word, i.e. include the space before it
       SCREAMING_SNAKE_CASE__					=					AddedToken(__lowerCamelCase			,     lstrip=__lowerCamelCase			,     rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase			,     __lowerCamelCase      ) else mask_token

       SCREAMING_SNAKE_CASE__					=					{} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=__lowerCamelCase			,     remove_space=__lowerCamelCase			,     keep_accents=__lowerCamelCase			,     bos_token=__lowerCamelCase			,     eos_token=__lowerCamelCase			,     unk_token=__lowerCamelCase			,     sep_token=__lowerCamelCase			,     pad_token=__lowerCamelCase			,     cls_token=__lowerCamelCase			,     mask_token=__lowerCamelCase			,     additional_special_tokens=__lowerCamelCase			,     sp_model_kwargs=self.sp_model_kwargs			,     **__lowerCamelCase			,     )

       SCREAMING_SNAKE_CASE__					=					3

       SCREAMING_SNAKE_CASE__					=					do_lower_case
       SCREAMING_SNAKE_CASE__					=					remove_space
       SCREAMING_SNAKE_CASE__					=					keep_accents
       SCREAMING_SNAKE_CASE__					=					vocab_file

       SCREAMING_SNAKE_CASE__					=					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
       self.sp_model.Load(__lowerCamelCase      )






  @property
  def    lowercase_	(							self		:	Optional[Any]      )	->							Union[str, Any]:
       return len(self.sp_model      )






  def    lowercase_	(							self		:	Tuple      )	->							Dict:
       SCREAMING_SNAKE_CASE__					=					{self.convert_ids_to_tokens(__lowerCamelCase      ): i for i in range(self.vocab_size      )}
       vocab.update(self.added_tokens_encoder      )
       return vocab






  def __getstate__(							self		:	List[Any]      )	->							List[Any]:
       SCREAMING_SNAKE_CASE__					=					self.__dict__.copy()
       SCREAMING_SNAKE_CASE__					=					None
       return state






  def __setstate__(							self		:	Any			,     __lowerCamelCase		:	Tuple      )	->							Tuple:
       SCREAMING_SNAKE_CASE__					=					d

       # for backward compatibility
       if not hasattr(self			,     '''sp_model_kwargs'''      ):
            SCREAMING_SNAKE_CASE__					=					{}

       SCREAMING_SNAKE_CASE__					=					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
       self.sp_model.Load(self.vocab_file      )






  def    lowercase_	(							self		:	Union[str, Any]			,     __lowerCamelCase		:	Union[str, Any]      )	->							List[str]:
       if self.remove_space:
            SCREAMING_SNAKE_CASE__					=					''' '''.join(inputs.strip().split()      )
       else:
            SCREAMING_SNAKE_CASE__					=					inputs
       SCREAMING_SNAKE_CASE__					=					outputs.replace('''``'''			,     '''"'''      ).replace('''\'\''''			,     '''"'''      )

       if not self.keep_accents:
            SCREAMING_SNAKE_CASE__					=					unicodedata.normalize('''NFKD'''			,     __lowerCamelCase      )
            SCREAMING_SNAKE_CASE__					=					''''''.join([c for c in outputs if not unicodedata.combining(__lowerCamelCase      )]      )
       if self.do_lower_case:
            SCREAMING_SNAKE_CASE__					=					outputs.lower()

       return outputs






  def    lowercase_	(							self		:	Any			,     __lowerCamelCase		:	str      )	->							List[str]:
       SCREAMING_SNAKE_CASE__					=					self.preprocess_text(__lowerCamelCase      )
       SCREAMING_SNAKE_CASE__					=					self.sp_model.encode(__lowerCamelCase			,     out_type=__lowerCamelCase      )
       SCREAMING_SNAKE_CASE__					=					[]
       for piece in pieces:
            if len(__lowerCamelCase      ) > 1 and piece[-1] == str(''','''      ) and piece[-2].isdigit():
                 SCREAMING_SNAKE_CASE__					=					self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCamelCase			,     ''''''      )      )
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                      if len(cur_pieces[0]      ) == 1:
                           SCREAMING_SNAKE_CASE__					=					cur_pieces[1:]
                      else:
                           SCREAMING_SNAKE_CASE__					=					cur_pieces[0][1:]
                 cur_pieces.append(piece[-1]      )
                 new_pieces.extend(__lowerCamelCase      )
            else:
                 new_pieces.append(__lowerCamelCase      )

       return new_pieces






  def    lowercase_	(							self		:	List[Any]			,     __lowerCamelCase		:	Optional[int]      )	->							Optional[Any]:
       return self.sp_model.PieceToId(__lowerCamelCase      )






  def    lowercase_	(							self		:	int			,     __lowerCamelCase		:	Tuple      )	->							List[Any]:
       return self.sp_model.IdToPiece(__lowerCamelCase      )






  def    lowercase_	(							self		:	int			,     __lowerCamelCase		:	Tuple      )	->							Optional[Any]:
       SCREAMING_SNAKE_CASE__					=					''''''.join(__lowerCamelCase      ).replace(__lowerCamelCase			,     ''' '''      ).strip()
       return out_string






  def    lowercase_	(							self		:	Union[str, Any]			,     __lowerCamelCase		:	List[int]			,     __lowerCamelCase		:	bool = False			,     __lowerCamelCase		:	bool = None			,     __lowerCamelCase		:	bool = True			,     **__lowerCamelCase		:	int			,     )	->							str:
       SCREAMING_SNAKE_CASE__					=					kwargs.pop('''use_source_tokenizer'''			,     __lowerCamelCase      )

       SCREAMING_SNAKE_CASE__					=					self.convert_ids_to_tokens(__lowerCamelCase			,     skip_special_tokens=__lowerCamelCase      )

       # To avoid mixing byte-level and unicode for byte-level BPT
       # we need to build string separately for added tokens and byte-level tokens
       # cf. https://github.com/huggingface/transformers/issues/1133
       SCREAMING_SNAKE_CASE__					=					[]
       SCREAMING_SNAKE_CASE__					=					[]
       for token in filtered_tokens:
            if skip_special_tokens and token in self.all_special_ids:
                 continue
            if token in self.added_tokens_encoder:
                 if current_sub_text:
                      sub_texts.append(self.convert_tokens_to_string(__lowerCamelCase      )      )
                      SCREAMING_SNAKE_CASE__					=					[]
                 sub_texts.append(__lowerCamelCase      )
            else:
                 current_sub_text.append(__lowerCamelCase      )
       if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(__lowerCamelCase      )      )

       # Mimic the behavior of the Rust tokenizer:
       # By default, there are no spaces between special tokens
       SCREAMING_SNAKE_CASE__					=					''''''.join(__lowerCamelCase      )

       SCREAMING_SNAKE_CASE__					=					(
           clean_up_tokenization_spaces
           if clean_up_tokenization_spaces is not None
           else self.clean_up_tokenization_spaces
       )
       if clean_up_tokenization_spaces:
            SCREAMING_SNAKE_CASE__					=					self.clean_up_tokenization(__lowerCamelCase      )
            return clean_text
       else:
            return text






  def    lowercase_	(							self		:	Dict			,     __lowerCamelCase		:	List[int]			,     __lowerCamelCase		:	Optional[List[int]] = None      )	->							List[int]:
       SCREAMING_SNAKE_CASE__					=					[self.sep_token_id]
       SCREAMING_SNAKE_CASE__					=					[self.cls_token_id]
       if token_ids_a is None:
            return token_ids_a + sep + cls
       return token_ids_a + sep + token_ids_a + sep + cls






  def    lowercase_	(							self		:	List[Any]			,     __lowerCamelCase		:	List[int]			,     __lowerCamelCase		:	Optional[List[int]] = None			,     __lowerCamelCase		:	bool = False      )	->							List[int]:

       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=__lowerCamelCase			,     token_ids_a=__lowerCamelCase			,     already_has_special_tokens=__lowerCamelCase      )

       if token_ids_a is not None:
            return ([0] * len(__lowerCamelCase      )) + [1] + ([0] * len(__lowerCamelCase      )) + [1, 1]
       return ([0] * len(__lowerCamelCase      )) + [1, 1]






  def    lowercase_	(							self		:	List[str]			,     __lowerCamelCase		:	List[int]			,     __lowerCamelCase		:	Optional[List[int]] = None      )	->							List[int]:
       SCREAMING_SNAKE_CASE__					=					[self.sep_token_id]
       SCREAMING_SNAKE_CASE__					=					[2]

       if token_ids_a is None:
            return len(token_ids_a + sep      ) * [0] + cls_segment_id
       return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1] + cls_segment_id







  def    lowercase_	(							self		:	str			,     __lowerCamelCase		:	str			,     __lowerCamelCase		:	Optional[str] = None      )	->							Tuple[str]:
       if not os.path.isdir(__lowerCamelCase      ):
            logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''      )
            return
       SCREAMING_SNAKE_CASE__					=					os.path.join(
           __lowerCamelCase			,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

       if os.path.abspath(self.vocab_file      ) != os.path.abspath(__lowerCamelCase      ) and os.path.isfile(self.vocab_file      ):
            copyfile(self.vocab_file			,     __lowerCamelCase      )
       elif not os.path.isfile(self.vocab_file      ):
            with open(__lowerCamelCase			,     '''wb'''      ) as fi:
                 SCREAMING_SNAKE_CASE__					=					self.sp_model.serialized_model_proto()
                 fi.write(__lowerCamelCase      )

       return (out_vocab_file,)


