






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCAmelCase__ :     int				    =    logging.get_logger(__name__)

UpperCAmelCase__ :     List[str]				    =    {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase__ :     Any				    =    {
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

UpperCAmelCase__ :     Dict				    =    {
    """junnyu/roformer_chinese_small""": 1_536,
    """junnyu/roformer_chinese_base""": 1_536,
    """junnyu/roformer_chinese_char_small""": 512,
    """junnyu/roformer_chinese_char_base""": 512,
    """junnyu/roformer_small_discriminator""": 128,
    """junnyu/roformer_small_generator""": 128,
}


UpperCAmelCase__ :     Optional[Any]				    =    {
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}







class        a__       (   lowercase_       ):







  """simple docstring"""




  UpperCAmelCase__      : Optional[Any]			       =VOCAB_FILES_NAMES
  UpperCAmelCase__      : Optional[int]			       =PRETRAINED_VOCAB_FILES_MAP
  UpperCAmelCase__      : Union[str, Any]			       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  UpperCAmelCase__      : Tuple			       =PRETRAINED_INIT_CONFIGURATION
  UpperCAmelCase__      : Dict			       =RoFormerTokenizer


  def __init__(			self    : Any				,				UpperCAmelCase__    : Dict=None				,				UpperCAmelCase__    : Union[str, Any]=None				,				UpperCAmelCase__    : Optional[Any]=True				,				UpperCAmelCase__    : Optional[Any]="[UNK]"				,				UpperCAmelCase__    : int="[SEP]"				,				UpperCAmelCase__    : int="[PAD]"				,				UpperCAmelCase__    : Dict="[CLS]"				,				UpperCAmelCase__    : List[str]="[MASK]"				,				UpperCAmelCase__    : Union[str, Any]=True				,				UpperCAmelCase__    : Optional[int]=None				,				**UpperCAmelCase__    : int				,				)					->List[Any]:
        """simple docstring"""







        super().__init__(
            lowerCamelCase_				,				tokenizer_file=lowerCamelCase_				,				do_lower_case=lowerCamelCase_				,				unk_token=lowerCamelCase_				,				sep_token=lowerCamelCase_				,				pad_token=lowerCamelCase_				,				cls_token=lowerCamelCase_				,				mask_token=lowerCamelCase_				,				tokenize_chinese_chars=lowerCamelCase_				,				strip_accents=lowerCamelCase_				,				**lowerCamelCase_				,				)

        SCREAMING_SNAKE_CASE :      Optional[int]       =   json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
        if (
            pre_tok_state.get("""lowercase"""				,				lowerCamelCase_		) != do_lower_case
            or pre_tok_state.get("""strip_accents"""				,				lowerCamelCase_		) != strip_accents
        ):
              SCREAMING_SNAKE_CASE :      Tuple       =   getattr(lowerCamelCase_				,				pre_tok_state.pop("""type"""		)		)
              SCREAMING_SNAKE_CASE :      Optional[Any]       =   do_lower_case
              SCREAMING_SNAKE_CASE :      Any       =   strip_accents
              SCREAMING_SNAKE_CASE :      List[Any]       =   pre_tok_class(**lowerCamelCase_		)

        SCREAMING_SNAKE_CASE :      Dict       =   do_lower_case


  def __getstate__(			self    : Dict		)					->int:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Dict       =   self.__dict__.copy()
        SCREAMING_SNAKE_CASE :      str       =   BertPreTokenizer()
        return state


  def __setstate__(			self    : Tuple				,				UpperCAmelCase__    : str		)					->int:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   d
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   self.__dict__["""_tokenizer"""].get_vocab()
        SCREAMING_SNAKE_CASE :      Tuple       =   PreTokenizer.custom(JiebaPreTokenizer(lowerCamelCase_		)		)


  def  _lowercase      (			self    : Optional[int]				,				UpperCAmelCase__    : Optional[int]				,				UpperCAmelCase__    : Tuple=None		)					->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Dict       =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

        return output


  def  _lowercase      (			self    : List[str]				,				UpperCAmelCase__    : List[int]				,				UpperCAmelCase__    : Optional[List[int]] = None		)					->List[str]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      List[str]       =   [self.sep_token_id]
        SCREAMING_SNAKE_CASE :      Optional[Any]       =   [self.cls_token_id]
        if token_ids_a is None:
              return len(cls + token_ids_a + sep		) * [0]
        return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]


  def  _lowercase      (			self    : Any				,				UpperCAmelCase__    : str				,				UpperCAmelCase__    : Optional[str] = None		)					->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      int       =   self._tokenizer.model.save(lowerCamelCase_				,				name=lowerCamelCase_		)
        return tuple(lowerCamelCase_		)




  def  _lowercase      (			self    : int				,				UpperCAmelCase__    : Tuple				,				UpperCAmelCase__    : List[Any]=None				,				UpperCAmelCase__    : Dict=None				,				UpperCAmelCase__    : List[Any]=False				,				**UpperCAmelCase__    : Union[str, Any]				,				)					->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Dict       =   BertPreTokenizer()
        return super().save_pretrained(lowerCamelCase_				,				lowerCamelCase_				,				lowerCamelCase_				,				lowerCamelCase_				,				**lowerCamelCase_		)







from typing import Any

import numpy as np






def 					__lowercase      (     _A			)     ->						bool:
      return np.array_equal(_A						,    matrix.conjugate().T			)






def 					__lowercase      (     _A						,    _A			)     ->						Any:
      SCREAMING_SNAKE_CASE :      Optional[int]       =   v.conjugate().T
      SCREAMING_SNAKE_CASE :      List[Any]       =   v_star.dot(_A			)
      assert isinstance(_A						,    np.ndarray			)
      return (v_star_dot.dot(_A			)) / (v_star.dot(_A			))






def 					__lowercase      (     )     ->						None:
      SCREAMING_SNAKE_CASE :      Any       =   np.array([[2, 2 + 1J, 4], [2 - 1J, 3, 1J], [4, -1J, 1]]			)
      SCREAMING_SNAKE_CASE :      Optional[int]       =   np.array([[1], [2], [3]]			)
      assert is_hermitian(_A			), F"{a} is not hermitian."
      print(rayleigh_quotient(_A						,    _A			)			)

      SCREAMING_SNAKE_CASE :      Any       =   np.array([[1, 2, 4], [2, 3, -1], [4, -1, 1]]			)
      assert is_hermitian(_A			), F"{a} is not hermitian."
      assert rayleigh_quotient(_A						,    _A			) == float(3			)


if __name__ == "__main__":
  import doctest

  doctest.testmod()
  tests()




