






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


__lowerCamelCase	:		List[str]										=   logging.get_logger(__name__)

__lowerCamelCase	:		int										=   {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCamelCase	:		Any										=   {
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

__lowerCamelCase	:		Optional[int]										=   {
    '''bert-base-uncased''': 5_12,
    '''bert-large-uncased''': 5_12,
    '''bert-base-cased''': 5_12,
    '''bert-large-cased''': 5_12,
    '''bert-base-multilingual-uncased''': 5_12,
    '''bert-base-multilingual-cased''': 5_12,
    '''bert-base-chinese''': 5_12,
    '''bert-base-german-cased''': 5_12,
    '''bert-large-uncased-whole-word-masking''': 5_12,
    '''bert-large-cased-whole-word-masking''': 5_12,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-base-cased-finetuned-mrpc''': 5_12,
    '''bert-base-german-dbmdz-cased''': 5_12,
    '''bert-base-german-dbmdz-uncased''': 5_12,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 5_12,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 5_12,
    '''wietsedv/bert-base-dutch-cased''': 5_12,
}

__lowerCamelCase	:		List[Any]										=   {
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}







class  _lowercase   (   snake_case__						):
   _a							:						List[str]										=							VOCAB_FILES_NAMES
   _a							:						Any										=							PRETRAINED_VOCAB_FILES_MAP
   _a							:						Dict										=							PRETRAINED_INIT_CONFIGURATION
   _a							:						List[str]										=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _a							:						List[str]										=							BertTokenizer
   def __init__(     self				,			a=None				,			a=None				,			a=True				,			a="[UNK]"				,			a="[SEP]"				,			a="[PAD]"				,			a="[CLS]"				,			a="[MASK]"				,			a=True				,			a=None				,			**a				,			):
    super().__init__(
        _A				,			tokenizer_file=_A				,			do_lower_case=_A				,			unk_token=_A				,			sep_token=_A				,			pad_token=_A				,			cls_token=_A				,			mask_token=_A				,			tokenize_chinese_chars=_A				,			strip_accents=_A				,			**_A				,			)

    snake_case__  :    Optional[int]							       =json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
    if (
        normalizer_state.get("""lowercase"""				,			_A   ) != do_lower_case
        or normalizer_state.get("""strip_accents"""				,			_A   ) != strip_accents
        or normalizer_state.get("""handle_chinese_chars"""				,			_A   ) != tokenize_chinese_chars
    ):
     snake_case__  :    Any							       =getattr(_A				,			normalizer_state.pop("""type"""   )   )
     snake_case__  :    Union[str, Any]							       =do_lower_case
     snake_case__  :    Optional[int]							       =strip_accents
     snake_case__  :    List[Any]							       =tokenize_chinese_chars
     snake_case__  :    int							       =normalizer_class(**_A   )

    snake_case__  :    Union[str, Any]							       =do_lower_case
   def 						lowercase__					(     self				,			a				,			a=None   ):
    snake_case__  :    Tuple							       =[self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

    return output
   def 						lowercase__					(     self				,			a				,			a = None   ):
    snake_case__  :    Optional[Any]							       =[self.sep_token_id]
    snake_case__  :    Optional[int]							       =[self.cls_token_id]
    if token_ids_a is None:
     return len(cls + token_ids_a + sep   ) * [0]
    return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



   def 						lowercase__					(     self				,			a				,			a = None   ):
    snake_case__  :    int							       =self._tokenizer.model.save(_A				,			name=_A   )
    return tuple(_A   )













import unittest

import numpy as np

from transformers import RoFormerConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
       import jax.numpy as jnp

       from transformers.models.roformer.modeling_flax_roformer import (
           FlaxRoFormerForMaskedLM,
           FlaxRoFormerForMultipleChoice,
           FlaxRoFormerForQuestionAnswering,
           FlaxRoFormerForSequenceClassification,
           FlaxRoFormerForTokenClassification,
           FlaxRoFormerModel,
       )







class  _lowercase   (   unittest.TestCase						):
      def __init__(     self				,			a				,			a=1_3				,			a=7				,			a=True				,			a=True				,			a=True				,			a=True				,			a=9_9				,			a=3_2				,			a=5				,			a=4				,			a=3_7				,			a="gelu"				,			a=0.1				,			a=0.1				,			a=5_1_2				,			a=1_6				,			a=2				,			a=0.02				,			a=4				,			):
       snake_case__  :    Any							       =parent
       snake_case__  :    Dict							       =batch_size
       snake_case__  :    List[Any]							       =seq_length
       snake_case__  :    str							       =is_training
       snake_case__  :    Union[str, Any]							       =use_attention_mask
       snake_case__  :    str							       =use_token_type_ids
       snake_case__  :    int							       =use_labels
       snake_case__  :    Tuple							       =vocab_size
       snake_case__  :    List[str]							       =hidden_size
       snake_case__  :    Dict							       =num_hidden_layers
       snake_case__  :    Optional[Any]							       =num_attention_heads
       snake_case__  :    List[str]							       =intermediate_size
       snake_case__  :    str							       =hidden_act
       snake_case__  :    Union[str, Any]							       =hidden_dropout_prob
       snake_case__  :    Tuple							       =attention_probs_dropout_prob
       snake_case__  :    Tuple							       =max_position_embeddings
       snake_case__  :    str							       =type_vocab_size
       snake_case__  :    Optional[Any]							       =type_sequence_label_size
       snake_case__  :    str							       =initializer_range
       snake_case__  :    List[Any]							       =num_choices
      def 						lowercase__					(     self   ):
       snake_case__  :    Union[str, Any]							       =ids_tensor([self.batch_size, self.seq_length]				,			self.vocab_size   )

       snake_case__  :    Tuple							       =None
       if self.use_attention_mask:
        snake_case__  :    Union[str, Any]							       =random_attention_mask([self.batch_size, self.seq_length]   )

       snake_case__  :    Union[str, Any]							       =None
       if self.use_token_type_ids:
        snake_case__  :    Optional[Any]							       =ids_tensor([self.batch_size, self.seq_length]				,			self.type_vocab_size   )

       snake_case__  :    Tuple							       =RoFormerConfig(
           vocab_size=self.vocab_size				,			hidden_size=self.hidden_size				,			num_hidden_layers=self.num_hidden_layers				,			num_attention_heads=self.num_attention_heads				,			intermediate_size=self.intermediate_size				,			hidden_act=self.hidden_act				,			hidden_dropout_prob=self.hidden_dropout_prob				,			attention_probs_dropout_prob=self.attention_probs_dropout_prob				,			max_position_embeddings=self.max_position_embeddings				,			type_vocab_size=self.type_vocab_size				,			is_decoder=a				,			initializer_range=self.initializer_range				,			)

       return config, input_ids, token_type_ids, attention_mask



      def 						lowercase__					(     self   ):
       snake_case__  :    Optional[int]							       =self.prepare_config_and_inputs()
       snake_case__       ,   snake_case__       ,   snake_case__       ,   snake_case__  :    Optional[Any]							       =config_and_inputs
       snake_case__  :    Dict							       ={"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
       return config, inputs_dict







@require_flax
class  _lowercase   (   _A						,				unittest.TestCase						):
      _a							:						str										=							True

      _a							:						Optional[Any]										=							(
          (
              FlaxRoFormerModel,
              FlaxRoFormerForMaskedLM,
              FlaxRoFormerForSequenceClassification,
              FlaxRoFormerForTokenClassification,
              FlaxRoFormerForMultipleChoice,
              FlaxRoFormerForQuestionAnswering,
          )
          if is_flax_available()
          else ()
      )
      def 						lowercase__					(     self   ):
       snake_case__  :    Optional[Any]							       =FlaxRoFormerModelTester(self   )



      @slow
      def 						lowercase__					(     self   ):
       for model_class_name in self.all_model_classes:
        snake_case__  :    Tuple							       =model_class_name.from_pretrained("""junnyu/roformer_chinese_small"""				,			from_pt=a   )
        snake_case__  :    List[Any]							       =model(np.ones((1, 1)   )   )
        self.assertIsNotNone(a   )







@require_flax
class  _lowercase   (   unittest.TestCase						):
      @slow
      def 						lowercase__					(     self   ):
       snake_case__  :    Optional[int]							       =FlaxRoFormerForMaskedLM.from_pretrained("""junnyu/roformer_chinese_base"""   )
       snake_case__  :    Tuple							       =jnp.array([[0, 1, 2, 3, 4, 5]]   )
       snake_case__  :    str							       =model(a   )[0]

       snake_case__  :    List[str]							       =5_0_0_0_0

       snake_case__  :    str							       =(1, 6, vocab_size)
       self.assertEqual(output.shape				,			a   )

       snake_case__  :    Optional[int]							       =jnp.array(
           [[[-0.1205, -1.0265, 0.2922], [-1.5134, 0.1974, 0.1519], [-5.0135, -3.9003, -0.8404]]]   )

       self.assertTrue(jnp.allclose(output[:, :3, :3]				,			a				,			atol=1e-4   )   )





