

"""simple docstring"""






import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
 import jax.numpy as jnp

 from transformers.models.albert.modeling_flax_albert import (
     FlaxAlbertForMaskedLM,
     FlaxAlbertForMultipleChoice,
     FlaxAlbertForPreTraining,
     FlaxAlbertForQuestionAnswering,
     FlaxAlbertForSequenceClassification,
     FlaxAlbertForTokenClassification,
     FlaxAlbertModel,
 )







class 	__UpperCamelCase    (    unittest.TestCase						):

  def __init__(		self    ,_A    ,_A=13    ,_A=7    ,_A=True    ,_A=True    ,_A=True    ,_A=True    ,_A=99    ,_A=32    ,_A=5    ,_A=4    ,_A=37    ,_A="gelu"    ,_A=0.1    ,_A=0.1    ,_A=512    ,_A=16    ,_A=2    ,_A=0.0_2    ,_A=4    ,):
      '''simple docstring'''

      _lowerCAmelCase       :   List[str]  =    parent
      _lowerCAmelCase       :   Optional[Any]  =    batch_size
      _lowerCAmelCase       :   str  =    seq_length
      _lowerCAmelCase       :   List[Any]  =    is_training
      _lowerCAmelCase       :   Dict  =    use_attention_mask
      _lowerCAmelCase       :   str  =    use_token_type_ids
      _lowerCAmelCase       :   Union[str, Any]  =    use_labels
      _lowerCAmelCase       :   Optional[int]  =    vocab_size
      _lowerCAmelCase       :   str  =    hidden_size
      _lowerCAmelCase       :   List[Any]  =    num_hidden_layers
      _lowerCAmelCase       :   List[str]  =    num_attention_heads
      _lowerCAmelCase       :   int  =    intermediate_size
      _lowerCAmelCase       :   Dict  =    hidden_act
      _lowerCAmelCase       :   List[str]  =    hidden_dropout_prob
      _lowerCAmelCase       :   Union[str, Any]  =    attention_probs_dropout_prob
      _lowerCAmelCase       :   Dict  =    max_position_embeddings
      _lowerCAmelCase       :   Dict  =    type_vocab_size
      _lowerCAmelCase       :   int  =    type_sequence_label_size
      _lowerCAmelCase       :   Any  =    initializer_range
      _lowerCAmelCase       :   Tuple  =    num_choices

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[str]  =    ids_tensor([self.batch_size, self.seq_length]    ,self.vocab_size      )

      _lowerCAmelCase       :   List[str]  =    None
      if self.use_attention_mask:
          _lowerCAmelCase       :   Optional[int]  =    random_attention_mask([self.batch_size, self.seq_length]      )

      _lowerCAmelCase       :   Dict  =    None
      if self.use_token_type_ids:
          _lowerCAmelCase       :   int  =    ids_tensor([self.batch_size, self.seq_length]    ,self.type_vocab_size      )

      _lowerCAmelCase       :   List[str]  =    AlbertConfig(
          vocab_size=self.vocab_size    ,hidden_size=self.hidden_size    ,num_hidden_layers=self.num_hidden_layers    ,num_attention_heads=self.num_attention_heads    ,intermediate_size=self.intermediate_size    ,hidden_act=self.hidden_act    ,hidden_dropout_prob=self.hidden_dropout_prob    ,attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,max_position_embeddings=self.max_position_embeddings    ,type_vocab_size=self.type_vocab_size    ,is_decoder=_A    ,initializer_range=self.initializer_range    ,)

      return config, input_ids, token_type_ids, attention_mask







  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Tuple  =    self.prepare_config_and_inputs()
      _lowerCAmelCase,					_lowerCAmelCase,					_lowerCAmelCase,					_lowerCAmelCase       :   Any  =    config_and_inputs
      _lowerCAmelCase       :   int  =    {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
      return config, inputs_dict







@require_flax
class 	__UpperCamelCase    (    a__							,							unittest.TestCase						):
  _UpperCAmelCase          =		(
      (
          FlaxAlbertModel,
          FlaxAlbertForPreTraining,
          FlaxAlbertForMaskedLM,
          FlaxAlbertForMultipleChoice,
          FlaxAlbertForQuestionAnswering,
          FlaxAlbertForSequenceClassification,
          FlaxAlbertForTokenClassification,
          FlaxAlbertForQuestionAnswering,
      )
      if is_flax_available()
      else ()
  )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Union[str, Any]  =    FlaxAlbertModelTester(self      )







  @slow
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      for model_class_name in self.all_model_classes:
          _lowerCAmelCase       :   Optional[Any]  =    model_class_name.from_pretrained('albert-base-v2'      )
          _lowerCAmelCase       :   Optional[int]  =    model(np.ones((1, 1)      )      )
          self.assertIsNotNone(_A      )







@require_flax
class 	__UpperCamelCase    (    unittest.TestCase						):

  @slow
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[Any]  =    FlaxAlbertModel.from_pretrained('albert-base-v2'      )
      _lowerCAmelCase       :   Any  =    np.array([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]      )
      _lowerCAmelCase       :   Dict  =    np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
      _lowerCAmelCase       :   List[Any]  =    model(_A    ,attention_mask=_A      )[0]
      _lowerCAmelCase       :   List[Any]  =    (1, 11, 768)
      self.assertEqual(output.shape    ,_A      )
      _lowerCAmelCase       :   str  =    np.array(
          [[[-0.6_5_1_3, 1.5_0_3_5, -0.2_7_6_6], [-0.6_5_1_5, 1.5_0_4_6, -0.2_7_8_0], [-0.6_5_1_2, 1.5_0_4_9, -0.2_7_8_4]]]      )

      self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]    ,_A    ,atol=1E-4      )      )








"""simple docstring"""






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_lowerCAmelCase						=		logging.get_logger(__name__)

_lowerCAmelCase						=		{"""vocab_file""": """spiece.model"""}

_lowerCAmelCase						=		{
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    }
}

_lowerCAmelCase						=		{
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

# Segments (not really needed)
_lowerCAmelCase						=		0
_lowerCAmelCase						=		1
_lowerCAmelCase						=		2
_lowerCAmelCase						=		3
_lowerCAmelCase						=		4







class 	__UpperCamelCase    (    a__						):
  _UpperCAmelCase          =		VOCAB_FILES_NAMES
  _UpperCAmelCase          =		PRETRAINED_VOCAB_FILES_MAP
  _UpperCAmelCase          =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _UpperCAmelCase          =		"left"

  def __init__(		self    ,_A    ,_A=False    ,_A=True    ,_A=False    ,_A="<s>"    ,_A="</s>"    ,_A="<unk>"    ,_A="<sep>"    ,_A="<pad>"    ,_A="<cls>"    ,_A="<mask>"    ,_A=["<eop>", "<eod>"]    ,_A = None    ,**_A    ,):
      '''simple docstring'''

      _lowerCAmelCase       :   Optional[Any]  =    AddedToken(_A    ,lstrip=_A    ,rstrip=_A      ) if isinstance(_A    ,_A      ) else mask_token

      _lowerCAmelCase       :   Union[str, Any]  =    {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=_A    ,remove_space=_A    ,keep_accents=_A    ,bos_token=_A    ,eos_token=_A    ,unk_token=_A    ,sep_token=_A    ,pad_token=_A    ,cls_token=_A    ,mask_token=_A    ,additional_special_tokens=_A    ,sp_model_kwargs=self.sp_model_kwargs    ,**_A    ,)

      _lowerCAmelCase       :   int  =    3

      _lowerCAmelCase       :   Union[str, Any]  =    do_lower_case
      _lowerCAmelCase       :   Dict  =    remove_space
      _lowerCAmelCase       :   int  =    keep_accents
      _lowerCAmelCase       :   List[str]  =    vocab_file

      _lowerCAmelCase       :   List[Any]  =    spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(_A      )

  @property
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      return len(self.sp_model      )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[Any]  =    {self.convert_ids_to_tokens(_A      ): i for i in range(self.vocab_size      )}
      vocab.update(self.added_tokens_encoder      )
      return vocab

  def __getstate__(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[Any]  =    self.__dict__.copy()
      _lowerCAmelCase       :   List[str]  =    None
      return state

  def __setstate__(		self    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Union[str, Any]  =    d

      # for backward compatibility
      if not hasattr(self    ,'sp_model_kwargs'      ):
          _lowerCAmelCase       :   Union[str, Any]  =    {}

      _lowerCAmelCase       :   Optional[Any]  =    spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(self.vocab_file      )

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      if self.remove_space:
          _lowerCAmelCase       :   str  =    ' '.join(inputs.strip().split()      )
      else:
          _lowerCAmelCase       :   Dict  =    inputs
      _lowerCAmelCase       :   List[str]  =    outputs.replace('``'    ,'"'      ).replace('\'\''    ,'"'      )

      if not self.keep_accents:
          _lowerCAmelCase       :   Optional[Any]  =    unicodedata.normalize('NFKD'    ,_A      )
          _lowerCAmelCase       :   Dict  =    ''.join([c for c in outputs if not unicodedata.combining(_A      )]      )
      if self.do_lower_case:
          _lowerCAmelCase       :   Tuple  =    outputs.lower()

      return outputs

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Union[str, Any]  =    self.preprocess_text(_A      )
      _lowerCAmelCase       :   int  =    self.sp_model.encode(_A    ,out_type=_A      )
      _lowerCAmelCase       :   int  =    []
      for piece in pieces:
          if len(_A      ) > 1 and piece[-1] == str(','      ) and piece[-2].isdigit():
              _lowerCAmelCase       :   Union[str, Any]  =    self.sp_model.EncodeAsPieces(piece[:-1].replace(_A    ,''      )      )
              if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                  if len(cur_pieces[0]      ) == 1:
                      _lowerCAmelCase       :   int  =    cur_pieces[1:]
                  else:
                      _lowerCAmelCase       :   Any  =    cur_pieces[0][1:]
              cur_pieces.append(piece[-1]      )
              new_pieces.extend(_A      )
          else:
              new_pieces.append(_A      )

      return new_pieces

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      return self.sp_model.PieceToId(_A      )

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      return self.sp_model.IdToPiece(_A      )

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Optional[Any]  =    ''.join(_A      ).replace(_A    ,' '      ).strip()
      return out_string

  def __lowerCamelCase		(		self    ,_A    ,_A = False    ,_A = None    ,_A = True    ,**_A    ,):
      '''simple docstring'''

      _lowerCAmelCase       :   Dict  =    kwargs.pop('use_source_tokenizer'    ,_A      )

      _lowerCAmelCase       :   Dict  =    self.convert_ids_to_tokens(_A    ,skip_special_tokens=_A      )

      # To avoid mixing byte-level and unicode for byte-level BPT
      # we need to build string separately for added tokens and byte-level tokens
      # cf. https://github.com/huggingface/transformers/issues/1133
      _lowerCAmelCase       :   Optional[Any]  =    []
      _lowerCAmelCase       :   int  =    []
      for token in filtered_tokens:
          if skip_special_tokens and token in self.all_special_ids:
              continue
          if token in self.added_tokens_encoder:
              if current_sub_text:
                  sub_texts.append(self.convert_tokens_to_string(_A      )      )
                  _lowerCAmelCase       :   Tuple  =    []
              sub_texts.append(_A      )
          else:
              current_sub_text.append(_A      )
      if current_sub_text:
          sub_texts.append(self.convert_tokens_to_string(_A      )      )

      # Mimic the behavior of the Rust tokenizer:
      # By default, there are no spaces between special tokens
      _lowerCAmelCase       :   List[Any]  =    ''.join(_A      )

      _lowerCAmelCase       :   Tuple  =    (
          clean_up_tokenization_spaces
          if clean_up_tokenization_spaces is not None
          else self.clean_up_tokenization_spaces
      )
      if clean_up_tokenization_spaces:
          _lowerCAmelCase       :   int  =    self.clean_up_tokenization(_A      )
          return clean_text
      else:
          return text

  def __lowerCamelCase		(		self    ,_A    ,_A = None      ):
      '''simple docstring'''

      _lowerCAmelCase       :   str  =    [self.sep_token_id]
      _lowerCAmelCase       :   Union[str, Any]  =    [self.cls_token_id]
      if token_ids_a is None:
          return token_ids_a + sep + cls
      return token_ids_a + sep + token_ids_a + sep + cls

  def __lowerCamelCase		(		self    ,_A    ,_A = None    ,_A = False      ):
      '''simple docstring'''

      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=_A    ,token_ids_a=_A    ,already_has_special_tokens=_A      )

      if token_ids_a is not None:
          return ([0] * len(_A      )) + [1] + ([0] * len(_A      )) + [1, 1]
      return ([0] * len(_A      )) + [1, 1]

  def __lowerCamelCase		(		self    ,_A    ,_A = None      ):
      '''simple docstring'''

      _lowerCAmelCase       :   str  =    [self.sep_token_id]
      _lowerCAmelCase       :   Any  =    [2]

      if token_ids_a is None:
          return len(token_ids_a + sep      ) * [0] + cls_segment_id
      return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1] + cls_segment_id







  def __lowerCamelCase		(		self    ,_A    ,_A = None      ):
      '''simple docstring'''

      if not os.path.isdir(_A      ):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
          return
      _lowerCAmelCase       :   str  =    os.path.join(
          _A    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

      if os.path.abspath(self.vocab_file      ) != os.path.abspath(_A      ) and os.path.isfile(self.vocab_file      ):
          copyfile(self.vocab_file    ,_A      )
      elif not os.path.isfile(self.vocab_file      ):
          with open(_A    ,'wb'      ) as fi:
              _lowerCAmelCase       :   Optional[Any]  =    self.sp_model.serialized_model_proto()
              fi.write(_A      )

      return (out_vocab_file,)





