
"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


__SCREAMING_SNAKE_CASE					:						Any				= logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE					:						Any				= {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__SCREAMING_SNAKE_CASE					:						List[str]				= {
    '''vocab_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

__SCREAMING_SNAKE_CASE					:						Union[str, Any]				= {
    '''yjernite/retribert-base-uncased''': 512,
}


__SCREAMING_SNAKE_CASE					:						str				= {
    '''yjernite/retribert-base-uncased''': {'''do_lower_case''': True},
}



class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				VOCAB_FILES_NAMES
			snake_case	      =				PRETRAINED_VOCAB_FILES_MAP
			snake_case	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case	      =				PRETRAINED_INIT_CONFIGURATION
			snake_case	      =				RetriBertTokenizer
			snake_case	      =				["input_ids", "attention_mask"]




			def __init__(				self						:					Tuple       ,   _snake_case						:					int=None       ,   _snake_case						:					Any=None       ,   _snake_case						:					str=True       ,   _snake_case						:					List[Any]="[UNK]"       ,   _snake_case						:					Optional[Any]="[SEP]"       ,   _snake_case						:					str="[PAD]"       ,   _snake_case						:					List[str]="[CLS]"       ,   _snake_case						:					List[str]="[MASK]"       ,   _snake_case						:					Dict=True       ,   _snake_case						:					Optional[int]=None       ,   **_snake_case						:					Tuple       ,   )      -> Union[str, Any]:







								"""simple docstring"""
								super().__init__(
								    _snake_case       ,   tokenizer_file=_snake_case       ,   do_lower_case=_snake_case       ,   unk_token=_snake_case       ,   sep_token=_snake_case       ,   pad_token=_snake_case       ,   cls_token=_snake_case       ,   mask_token=_snake_case       ,   tokenize_chinese_chars=_snake_case       ,   strip_accents=_snake_case       ,   **_snake_case       ,   )

								A_				 =						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
								if (
								    normalizer_state.get("lowercase"       ,   _snake_case	) != do_lower_case
								    or normalizer_state.get("strip_accents"       ,   _snake_case	) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"       ,   _snake_case	) != tokenize_chinese_chars
								):
													A_				 =						getattr(_snake_case       ,   normalizer_state.pop("type"	)	)
													A_				 =						do_lower_case
													A_				 =						strip_accents
													A_				 =						tokenize_chinese_chars
													A_				 =						normalizer_class(**_snake_case	)

								A_				 =						do_lower_case




			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					List[Any]       ,   _snake_case						:					Union[str, Any]=None	)      -> List[Any]:







								"""simple docstring"""
								A_				 =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

								return output




			def      lowerCamelCase__     (				self						:					Any       ,   _snake_case						:					List[int]       ,   _snake_case						:					Optional[List[int]] = None	)      -> List[int]:







								"""simple docstring"""
								A_				 =						[self.sep_token_id]
								A_				 =						[self.cls_token_id]
								if token_ids_a is None:
													return len(cls + token_ids_a + sep	) * [0]
								return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					str       ,   _snake_case						:					Optional[str] = None	)      -> Tuple[str]:







								"""simple docstring"""
								A_				 =						self._tokenizer.model.save(_snake_case       ,   name=_snake_case	)
								return tuple(_snake_case	)






"""simple docstring"""





import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
UpperCamelCase_					:						Optional[Any]				= '''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
UpperCamelCase_					:						Union[str, Any]				= direct_transformers_import(PATH_TO_TRANSFORMERS)

UpperCamelCase_					:						List[Any]				= transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
UpperCamelCase_					:						str				= re.compile(R'''\[(.+?)\]\((https://huggingface\.co/.+?)\)''')


UpperCamelCase_					:						Dict				= {
    '''DecisionTransformerConfig''',
    '''EncoderDecoderConfig''',
    '''MusicgenConfig''',
    '''RagConfig''',
    '''SpeechEncoderDecoderConfig''',
    '''TimmBackboneConfig''',
    '''VisionEncoderDecoderConfig''',
    '''VisionTextDualEncoderConfig''',
    '''LlamaConfig''',
}
def 							A_     (__a				):

					'''simple docstring'''





					A_				 =						None

					# source code of `config_class`
					A_				 =						inspect.getsource(__a				)
					A_				 =						_re_checkpoint.findall(__a				)

					# Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
					# For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
					for ckpt_name, ckpt_link in checkpoints:
										# allow the link to end with `/`
										if ckpt_link.endswith("/"				):
															A_				 =						ckpt_link[:-1]

										# verify the checkpoint name corresponds to the checkpoint link
										A_				 =						f'https://huggingface.co/{ckpt_name}'
										if ckpt_link == ckpt_link_from_name:
															A_				 =						ckpt_name
															break

					return checkpoint
def 							A_     ():

					'''simple docstring'''





					A_				 =						[]

					for config_class in list(CONFIG_MAPPING.values()				):
										# Skip deprecated models
										if "models.deprecated" in config_class.__module__:
															continue
										A_				 =						get_checkpoint_from_config_class(__a				)

										A_				 =						config_class.__name__
										if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
															configs_without_checkpoint.append(__a				)

					if len(__a				) > 0:
										A_				 =						"\n".join(sorted(__a				)				)
										raise ValueError(f'The following configurations don\'t contain any valid checkpoint:\n{message}'				)


if __name__ == "__main__":
		check_config_docstrings_have_checkpoints()




