
"""simple docstring"""





import unittest

import numpy as np

from transformers import DistilBertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
		import jax.numpy as jnp

		from transformers.models.distilbert.modeling_flax_distilbert import (
		    FlaxDistilBertForMaskedLM,
		    FlaxDistilBertForMultipleChoice,
		    FlaxDistilBertForQuestionAnswering,
		    FlaxDistilBertForSequenceClassification,
		    FlaxDistilBertForTokenClassification,
		    FlaxDistilBertModel,
		)



class __lowerCAmelCase				(					unittest.TestCase      ):




			"""simple docstring"""







			def __init__(				self						:					Dict       ,   _snake_case						:					Any       ,   _snake_case						:					List[str]=13       ,   _snake_case						:					List[str]=7       ,   _snake_case						:					int=True       ,   _snake_case						:					Dict=True       ,   _snake_case						:					Dict=True       ,   _snake_case						:					List[Any]=True       ,   _snake_case						:					Union[str, Any]=99       ,   _snake_case						:					List[str]=32       ,   _snake_case						:					Optional[int]=5       ,   _snake_case						:					int=4       ,   _snake_case						:					str=37       ,   _snake_case						:					Union[str, Any]="gelu"       ,   _snake_case						:					int=0.1       ,   _snake_case						:					Union[str, Any]=0.1       ,   _snake_case						:					Optional[Any]=512       ,   _snake_case						:					str=16       ,   _snake_case						:					str=2       ,   _snake_case						:					List[str]=0.0_2       ,   _snake_case						:					List[Any]=4       ,   )      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						parent
								A_				 =						batch_size
								A_				 =						seq_length
								A_				 =						is_training
								A_				 =						use_attention_mask
								A_				 =						use_token_type_ids
								A_				 =						use_labels
								A_				 =						vocab_size
								A_				 =						hidden_size
								A_				 =						num_hidden_layers
								A_				 =						num_attention_heads
								A_				 =						intermediate_size
								A_				 =						hidden_act
								A_				 =						hidden_dropout_prob
								A_				 =						attention_probs_dropout_prob
								A_				 =						max_position_embeddings
								A_				 =						type_vocab_size
								A_				 =						type_sequence_label_size
								A_				 =						initializer_range
								A_				 =						num_choices




			def      lowerCamelCase__     (				self						:					Optional[int]	)      -> Optional[int]:







								"""simple docstring"""
								A_				 =						ids_tensor([self.batch_size, self.seq_length]       ,   self.vocab_size	)

								A_				 =						None
								if self.use_attention_mask:
													A_				 =						random_attention_mask([self.batch_size, self.seq_length]	)

								A_				 =						DistilBertConfig(
								    vocab_size=self.vocab_size       ,   dim=self.hidden_size       ,   n_layers=self.num_hidden_layers       ,   n_heads=self.num_attention_heads       ,   hidden_dim=self.intermediate_size       ,   hidden_act=self.hidden_act       ,   dropout=self.hidden_dropout_prob       ,   attention_dropout=self.attention_probs_dropout_prob       ,   max_position_embeddings=self.max_position_embeddings       ,   initializer_range=self.initializer_range       ,   tie_weights_=_snake_case       ,   )

								return config, input_ids, attention_mask






			def      lowerCamelCase__     (				self						:					Dict	)      -> Dict:







								"""simple docstring"""
								A_				 =						self.prepare_config_and_inputs()
								A_ ,    A_ ,    A_				 =						config_and_inputs
								A_				 =						{"input_ids": input_ids, "attention_mask": attention_mask}
								return config, inputs_dict





@require_flax
class __lowerCAmelCase				(					_lowercase     ,				unittest.TestCase      ):




			"""simple docstring"""







			snake_case	      =				(
			    (
			        FlaxDistilBertModel,
			        FlaxDistilBertForMaskedLM,
			        FlaxDistilBertForMultipleChoice,
			        FlaxDistilBertForQuestionAnswering,
			        FlaxDistilBertForSequenceClassification,
			        FlaxDistilBertForTokenClassification,
			        FlaxDistilBertForQuestionAnswering,
			    )
			    if is_flax_available()
			    else ()
			)




			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> Any:







								"""simple docstring"""
								A_				 =						FlaxDistilBertModelTester(self	)






			@slow
			def      lowerCamelCase__     (				self						:					Any	)      -> Optional[int]:







								"""simple docstring"""
								for model_class_name in self.all_model_classes:
													A_				 =						model_class_name.from_pretrained("distilbert-base-uncased"	)
													A_				 =						model(np.ones((1, 1)	)	)
													self.assertIsNotNone(_snake_case	)





@require_flax
class __lowerCAmelCase				(					unittest.TestCase      ):




			"""simple docstring"""







			@slow
			def      lowerCamelCase__     (				self						:					Tuple	)      -> List[Any]:







								"""simple docstring"""
								A_				 =						FlaxDistilBertModel.from_pretrained("distilbert-base-uncased"	)
								A_				 =						np.array([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]	)
								A_				 =						np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
								A_				 =						model(_snake_case       ,   attention_mask=_snake_case	)[0]
								A_				 =						(1, 11, 768)
								self.assertEqual(output.shape       ,   _snake_case	)
								A_				 =						np.array([[[-0.1_6_3_9, 0.3_2_9_9, 0.1_6_4_8], [-0.1_7_4_6, 0.3_2_8_9, 0.1_7_1_0], [-0.1_8_8_4, 0.3_3_5_7, 0.1_8_1_0]]]	)

								self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]       ,   _snake_case       ,   atol=1e-4	)	)






"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


UpperCamelCase_					:						Any				= logging.get_logger(__name__)

UpperCamelCase_					:						Any				= {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_					:						List[str]				= {
    '''vocab_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase_					:						Union[str, Any]				= {
    '''yjernite/retribert-base-uncased''': 512,
}


UpperCamelCase_					:						str				= {
    '''yjernite/retribert-base-uncased''': {'''do_lower_case''': True},
}



class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				VOCAB_FILES_NAMES
			snake_case	      =				PRETRAINED_VOCAB_FILES_MAP
			snake_case	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case	      =				PRETRAINED_INIT_CONFIGURATION
			snake_case	      =				RetriBertTokenizer
			snake_case	      =				["input_ids", "attention_mask"]




			def __init__(				self						:					Tuple       ,   _snake_case						:					int=None       ,   _snake_case						:					Any=None       ,   _snake_case						:					str=True       ,   _snake_case						:					List[Any]="[UNK]"       ,   _snake_case						:					Optional[Any]="[SEP]"       ,   _snake_case						:					str="[PAD]"       ,   _snake_case						:					List[str]="[CLS]"       ,   _snake_case						:					List[str]="[MASK]"       ,   _snake_case						:					Dict=True       ,   _snake_case						:					Optional[int]=None       ,   **_snake_case						:					Tuple       ,   )      -> Union[str, Any]:







								"""simple docstring"""
								super().__init__(
								    _snake_case       ,   tokenizer_file=_snake_case       ,   do_lower_case=_snake_case       ,   unk_token=_snake_case       ,   sep_token=_snake_case       ,   pad_token=_snake_case       ,   cls_token=_snake_case       ,   mask_token=_snake_case       ,   tokenize_chinese_chars=_snake_case       ,   strip_accents=_snake_case       ,   **_snake_case       ,   )

								A_				 =						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
								if (
								    normalizer_state.get("lowercase"       ,   _snake_case	) != do_lower_case
								    or normalizer_state.get("strip_accents"       ,   _snake_case	) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"       ,   _snake_case	) != tokenize_chinese_chars
								):
													A_				 =						getattr(_snake_case       ,   normalizer_state.pop("type"	)	)
													A_				 =						do_lower_case
													A_				 =						strip_accents
													A_				 =						tokenize_chinese_chars
													A_				 =						normalizer_class(**_snake_case	)

								A_				 =						do_lower_case




			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					List[Any]       ,   _snake_case						:					Union[str, Any]=None	)      -> List[Any]:







								"""simple docstring"""
								A_				 =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

								return output




			def      lowerCamelCase__     (				self						:					Any       ,   _snake_case						:					List[int]       ,   _snake_case						:					Optional[List[int]] = None	)      -> List[int]:







								"""simple docstring"""
								A_				 =						[self.sep_token_id]
								A_				 =						[self.cls_token_id]
								if token_ids_a is None:
													return len(cls + token_ids_a + sep	) * [0]
								return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					str       ,   _snake_case						:					Optional[str] = None	)      -> Tuple[str]:







								"""simple docstring"""
								A_				 =						self._tokenizer.model.save(_snake_case       ,   name=_snake_case	)
								return tuple(_snake_case	)




