
"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_snake_case          =						logging.get_logger(__name__)

_snake_case          =						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_snake_case          =						{
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

_snake_case          =						{
    '''google/realm-cc-news-pretrained-embedder''': 5_1_2,
    '''google/realm-cc-news-pretrained-encoder''': 5_1_2,
    '''google/realm-cc-news-pretrained-scorer''': 5_1_2,
    '''google/realm-cc-news-pretrained-openqa''': 5_1_2,
    '''google/realm-orqa-nq-openqa''': 5_1_2,
    '''google/realm-orqa-nq-reader''': 5_1_2,
    '''google/realm-orqa-wq-openqa''': 5_1_2,
    '''google/realm-orqa-wq-reader''': 5_1_2,
}

_snake_case          =						{
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}
class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							Union[str, Any]							=					VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_:							List[Any]							=					PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_:							List[str]							=					PRETRAINED_INIT_CONFIGURATION
				SCREAMING_SNAKE_CASE_:							str							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_:							Dict							=					RealmTokenizer


				def __init__(					self	:			Optional[Any]				,     UpperCAmelCase_	:			Optional[int]=None				,     UpperCAmelCase_	:			Optional[int]=None				,     UpperCAmelCase_	:			str=True				,     UpperCAmelCase_	:			Union[str, Any]="[UNK]"				,     UpperCAmelCase_	:			Tuple="[SEP]"				,     UpperCAmelCase_	:			Any="[PAD]"				,     UpperCAmelCase_	:			List[str]="[CLS]"				,     UpperCAmelCase_	:			List[Any]="[MASK]"				,     UpperCAmelCase_	:			List[str]=True				,     UpperCAmelCase_	:			str=None				,     **UpperCAmelCase_	:			Any				,     )		->  str:







											"""simple docstring"""




											super().__init__(
											    UpperCAmelCase_				,     tokenizer_file=UpperCAmelCase_				,     do_lower_case=UpperCAmelCase_				,     unk_token=UpperCAmelCase_				,     sep_token=UpperCAmelCase_				,     pad_token=UpperCAmelCase_				,     cls_token=UpperCAmelCase_				,     mask_token=UpperCAmelCase_				,     tokenize_chinese_chars=UpperCAmelCase_				,     strip_accents=UpperCAmelCase_				,     **UpperCAmelCase_				,     )

											_lowerCAmelCase  	=			json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
											if (
											    normalizer_state.get('lowercase'				,     UpperCAmelCase_  ) != do_lower_case
											    or normalizer_state.get('strip_accents'				,     UpperCAmelCase_  ) != strip_accents
											    or normalizer_state.get('handle_chinese_chars'				,     UpperCAmelCase_  ) != tokenize_chinese_chars
											):
																		_lowerCAmelCase  	=			getattr(UpperCAmelCase_				,     normalizer_state.pop('type'  )  )
																		_lowerCAmelCase  	=			do_lower_case
																		_lowerCAmelCase  	=			strip_accents
																		_lowerCAmelCase  	=			tokenize_chinese_chars
																		_lowerCAmelCase  	=			normalizer_class(**UpperCAmelCase_  )

											_lowerCAmelCase  	=			do_lower_case


				def 				__lowerCamelCase	(					self	:			Tuple				,     UpperCAmelCase_	:			Tuple				,     **UpperCAmelCase_	:			Any  )		->  Dict:







											"""simple docstring"""




											_lowerCAmelCase  	=			PaddingStrategy.MAX_LENGTH

											_lowerCAmelCase  	=			text
											_lowerCAmelCase  	=			kwargs.pop('text_pair'				,     UpperCAmelCase_  )
											_lowerCAmelCase  	=			kwargs.pop('return_tensors'				,     UpperCAmelCase_  )

											_lowerCAmelCase  	=			{
											    'input_ids': [],
											    'attention_mask': [],
											    'token_type_ids': [],
											}

											for idx, candidate_text in enumerate(UpperCAmelCase_  ):
																		if batch_text_pair is not None:
																									_lowerCAmelCase  	=			batch_text_pair[idx]
																		else:
																									_lowerCAmelCase  	=			None

																		_lowerCAmelCase  	=			super().__call__(UpperCAmelCase_				,     UpperCAmelCase_				,     return_tensors=UpperCAmelCase_				,     **UpperCAmelCase_  )

																		_lowerCAmelCase  	=			encoded_candidates.get('input_ids'  )
																		_lowerCAmelCase  	=			encoded_candidates.get('attention_mask'  )
																		_lowerCAmelCase  	=			encoded_candidates.get('token_type_ids'  )

																		if encoded_input_ids is not None:
																									output_data["input_ids"].append(UpperCAmelCase_  )
																		if encoded_attention_mask is not None:
																									output_data["attention_mask"].append(UpperCAmelCase_  )
																		if encoded_token_type_ids is not None:
																									output_data["token_type_ids"].append(UpperCAmelCase_  )

											_lowerCAmelCase  	=			{key: item for key, item in output_data.items() if len(UpperCAmelCase_  ) != 0}

											return BatchEncoding(UpperCAmelCase_				,     tensor_type=UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			Optional[Any]				,     UpperCAmelCase_	:			List[Any]				,     UpperCAmelCase_	:			List[str]=None  )		->  Union[str, Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

											return output


				def 				__lowerCamelCase	(					self	:			Union[str, Any]				,     UpperCAmelCase_	:			List[int]				,     UpperCAmelCase_	:			Optional[List[int]] = None  )		->  List[int]:







											"""simple docstring"""




											_lowerCAmelCase  	=			[self.sep_token_id]
											_lowerCAmelCase  	=			[self.cls_token_id]
											if token_ids_a is None:
																		return len(cls + token_ids_a + sep  ) * [0]
											return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







				def 				__lowerCamelCase	(					self	:			Optional[int]				,     UpperCAmelCase_	:			str				,     UpperCAmelCase_	:			Optional[str] = None  )		->  Tuple[str]:







											"""simple docstring"""




											_lowerCAmelCase  	=			self._tokenizer.model.save(UpperCAmelCase_				,     name=UpperCAmelCase_  )
											return tuple(UpperCAmelCase_  )








"""simple docstring"""



import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase      ,				unittest.TestCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							str							=					None
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							Any							=					True
				SCREAMING_SNAKE_CASE_:							List[Any]							=					False
				SCREAMING_SNAKE_CASE_:							Optional[int]							=					"tokenizer_file"
				SCREAMING_SNAKE_CASE_:							str							=					{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}


				def 				__lowerCamelCase	(					self	:			Tuple  )		->  str:







											"""simple docstring"""




											super().setUp()
											_lowerCAmelCase  	=			BloomTokenizerFast.from_pretrained('bigscience/tokenizer'  )
											tokenizer.save_pretrained(self.tmpdirname  )


				def 				__lowerCamelCase	(					self	:			List[str]				,     **UpperCAmelCase_	:			Any  )		->  Optional[Any]:







											"""simple docstring"""




											kwargs.update(self.special_tokens_map  )
											return BloomTokenizerFast.from_pretrained(self.tmpdirname				,     **UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  str:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()

											_lowerCAmelCase  	=			['The quick brown fox</s>', 'jumps over the lazy dog</s>']
											_lowerCAmelCase  	=			[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

											_lowerCAmelCase  	=			tokenizer.batch_encode_plus(UpperCAmelCase_  )['input_ids']
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )

											_lowerCAmelCase  	=			tokenizer.batch_decode(UpperCAmelCase_  )
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			Any				,     UpperCAmelCase_	:			List[str]=6  )		->  Any:







											"""simple docstring"""




											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																									_lowerCAmelCase  	=			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_  )
																									# tokenizer_r.pad_token = None # Hotfixing padding = None
																									# Simple input
																									_lowerCAmelCase  	=			'This is a simple input'
																									_lowerCAmelCase  	=			['This is a simple input 1', 'This is a simple input 2']
																									_lowerCAmelCase  	=			('This is a simple input', 'This is a pair')
																									_lowerCAmelCase  	=			[
																									    ('This is a simple input 1', 'This is a simple input 2'),
																									    ('This is a simple pair 1', 'This is a simple pair 2'),
																									]

																									# Simple input tests
																									try:
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )

																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																									except ValueError:
																																self.fail('Bloom Tokenizer should be able to deal with padding'  )

																									_lowerCAmelCase  	=			None  # Hotfixing padding = None
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )


				def 				__lowerCamelCase	(					self	:			str  )		->  Any:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()
											_lowerCAmelCase  	=			load_dataset('xnli'				,     'all_languages'				,     split='test'				,     streaming=UpperCAmelCase_  )

											_lowerCAmelCase  	=			next(iter(UpperCAmelCase_  )  )['premise']  # pick up one data
											_lowerCAmelCase  	=			list(sample_data.values()  )

											_lowerCAmelCase  	=			list(map(tokenizer.encode				,     UpperCAmelCase_  )  )
											_lowerCAmelCase  	=			[tokenizer.decode(UpperCAmelCase_				,     clean_up_tokenization_spaces=UpperCAmelCase_  ) for x in output_tokens]
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )







				def 				__lowerCamelCase	(					self	:			Tuple  )		->  List[Any]:







											"""simple docstring"""




											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  )				,     1  )
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  )				,     1  )






