





"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


UpperCamelCase_					:						Optional[Any]				= logging.get_logger(__name__)

UpperCamelCase_					:						Union[str, Any]				= {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_					:						Tuple				= {
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase_					:						List[str]				= {
    '''bert-base-uncased''': 512,
    '''bert-large-uncased''': 512,
    '''bert-base-cased''': 512,
    '''bert-large-cased''': 512,
    '''bert-base-multilingual-uncased''': 512,
    '''bert-base-multilingual-cased''': 512,
    '''bert-base-chinese''': 512,
    '''bert-base-german-cased''': 512,
    '''bert-large-uncased-whole-word-masking''': 512,
    '''bert-large-cased-whole-word-masking''': 512,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 512,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 512,
    '''bert-base-cased-finetuned-mrpc''': 512,
    '''bert-base-german-dbmdz-cased''': 512,
    '''bert-base-german-dbmdz-uncased''': 512,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 512,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 512,
    '''wietsedv/bert-base-dutch-cased''': 512,
}

UpperCamelCase_					:						Union[str, Any]				= {
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}



class __lowerCAmelCase				(					lowercase__      ):




			"""simple docstring"""







			snake_case	      =				VOCAB_FILES_NAMES
			snake_case	      =				PRETRAINED_VOCAB_FILES_MAP
			snake_case	      =				PRETRAINED_INIT_CONFIGURATION
			snake_case	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case	      =				BertTokenizer




			def __init__(				self						:					Optional[Any]       ,   _snake_case						:					List[str]=None       ,   _snake_case						:					Optional[int]=None       ,   _snake_case						:					Union[str, Any]=True       ,   _snake_case						:					Optional[Any]="[UNK]"       ,   _snake_case						:					Union[str, Any]="[SEP]"       ,   _snake_case						:					Dict="[PAD]"       ,   _snake_case						:					Union[str, Any]="[CLS]"       ,   _snake_case						:					Optional[int]="[MASK]"       ,   _snake_case						:					str=True       ,   _snake_case						:					List[Any]=None       ,   **_snake_case						:					Any       ,   )      -> Any:







								"""simple docstring"""
								super().__init__(
								    _snake_case       ,   tokenizer_file=_snake_case       ,   do_lower_case=_snake_case       ,   unk_token=_snake_case       ,   sep_token=_snake_case       ,   pad_token=_snake_case       ,   cls_token=_snake_case       ,   mask_token=_snake_case       ,   tokenize_chinese_chars=_snake_case       ,   strip_accents=_snake_case       ,   **_snake_case       ,   )

								A_				 =						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
								if (
								    normalizer_state.get("lowercase"       ,   _snake_case	) != do_lower_case
								    or normalizer_state.get("strip_accents"       ,   _snake_case	) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"       ,   _snake_case	) != tokenize_chinese_chars
								):
													A_				 =						getattr(_snake_case       ,   normalizer_state.pop("type"	)	)
													A_				 =						do_lower_case
													A_				 =						strip_accents
													A_				 =						tokenize_chinese_chars
													A_				 =						normalizer_class(**_snake_case	)

								A_				 =						do_lower_case




			def      lowerCamelCase__     (				self						:					List[Any]       ,   _snake_case						:					List[Any]       ,   _snake_case						:					str=None	)      -> str:







								"""simple docstring"""
								A_				 =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

								return output




			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					Optional[Any]       ,   _snake_case						:					Any = None	)      -> Tuple:







								"""simple docstring"""
								A_				 =						[self.sep_token_id]
								A_				 =						[self.cls_token_id]
								if token_ids_a is None:
													return len(cls + token_ids_a + sep	) * [0]
								return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






			def      lowerCamelCase__     (				self						:					Any       ,   _snake_case						:					List[str]       ,   _snake_case						:					Tuple = None	)      -> str:







								"""simple docstring"""
								A_				 =						self._tokenizer.model.save(_snake_case       ,   name=_snake_case	)
								return tuple(_snake_case	)






"""simple docstring"""





from __future__ import annotations
def 							A_     (__a				,   __a = None				,   __a = None				,   __a = False				,   ):

					'''simple docstring'''





					A_				 =						cipher_alphabet or [chr(__a				) for i in range(97				,   123				)]

					# If the argument is None or the user provided an empty dictionary
					if not frequencies_dict:
										# Frequencies of letters in the english language (how much they show up)
										A_				 =						{
										    "a": 0.08497,
										    "b": 0.01492,
										    "c": 0.02202,
										    "d": 0.04253,
										    "e": 0.11162,
										    "f": 0.02228,
										    "g": 0.02015,
										    "h": 0.06094,
										    "i": 0.07546,
										    "j": 0.00153,
										    "k": 0.01292,
										    "l": 0.04025,
										    "m": 0.02406,
										    "n": 0.06749,
										    "o": 0.07507,
										    "p": 0.01929,
										    "q": 0.00095,
										    "r": 0.07587,
										    "s": 0.06327,
										    "t": 0.09356,
										    "u": 0.02758,
										    "v": 0.00978,
										    "w": 0.02560,
										    "x": 0.00150,
										    "y": 0.01994,
										    "z": 0.00077,
										}
					else:
										# Custom frequencies dictionary
										A_				 =						frequencies_dict

					if not case_sensitive:
										A_				 =						ciphertext.lower()

					# Chi squared statistic values
					A_				 =						{}

					# cycle through all of the shifts
					for shift in range(len(__a				)				):
										A_				 =						""

										# decrypt the message with the shift
										for letter in ciphertext:
															try:
																				# Try to index the letter in the alphabet
																				A_				 =						(alphabet_letters.index(letter.lower()				) - shift) % len(
																				    __a				)
																				decrypted_with_shift += (
																				    alphabet_letters[new_key].upper()
																				    if case_sensitive and letter.isupper()
																				    else alphabet_letters[new_key]
																				)
															except ValueError:
																				# Append the character if it isn't in the alphabet
																				decrypted_with_shift += letter

										A_				 =						0.0

										# Loop through each letter in the decoded message with the shift
										for letter in decrypted_with_shift:
															if case_sensitive:
																				A_				 =						letter.lower()
																				if letter in frequencies:
																									# Get the amount of times the letter occurs in the message
																									A_				 =						decrypted_with_shift.lower().count(__a				)

																									# Get the excepcted amount of times the letter should appear based
																									# on letter frequencies
																									A_				 =						frequencies[letter] * occurrences

																									# Complete the chi squared statistic formula
																									A_				 =						((occurrences - expected) ** 2) / expected

																									# Add the margin of error to the total chi squared statistic
																									chi_squared_statistic += chi_letter_value
															else:
																				if letter.lower() in frequencies:
																									# Get the amount of times the letter occurs in the message
																									A_				 =						decrypted_with_shift.count(__a				)

																									# Get the excepcted amount of times the letter should appear based
																									# on letter frequencies
																									A_				 =						frequencies[letter] * occurrences

																									# Complete the chi squared statistic formula
																									A_				 =						((occurrences - expected) ** 2) / expected

																									# Add the margin of error to the total chi squared statistic
																									chi_squared_statistic += chi_letter_value

        # Add the data to the chi_squared_statistic_values dictionary
										A_				 =						(
										    chi_squared_statistic,
										    decrypted_with_shift,
										)

					# Get the most likely cipher by finding the cipher with the smallest chi squared
					# statistic
					def chi_squared_statistic_values_sorting_key(__a				) -> tuple[float, str]:
										return chi_squared_statistic_values[key]

					A_				 =						min(
					    __a				,   key=__a				,   )

					# Get all the data from the most likely cipher (key, decoded message)
					(
					    (
					    A_
					) ,    (
					    A_
					) ,    
					)				 =						chi_squared_statistic_values[most_likely_cipher]

					# Return the data on the most likely shift
					return (
					    most_likely_cipher,
					    most_likely_cipher_chi_squared_value,
					    decoded_most_likely_cipher,
					)




