
"""simple docstring"""





import argparse
import os
import re

import packaging.version


UpperCamelCase_					:						Tuple				= '''examples/'''
UpperCamelCase_					:						List[Any]				= {
    '''examples''': (re.compile(R'''^check_min_version\("[^"]+"\)\s*$''', re.MULTILINE), '''check_min_version("VERSION")\n'''),
    '''init''': (re.compile(R'''^__version__\s+=\s+"([^"]+)"\s*$''', re.MULTILINE), '''__version__ = "VERSION"\n'''),
    '''setup''': (re.compile(R'''^(\s*)version\s*=\s*"[^"]+",''', re.MULTILINE), R'''\1version="VERSION",'''),
    '''doc''': (re.compile(R'''^(\s*)release\s*=\s*"[^"]+"$''', re.MULTILINE), '''release = "VERSION"\n'''),
}
UpperCamelCase_					:						Optional[Any]				= {
    '''init''': '''src/transformers/__init__.py''',
    '''setup''': '''setup.py''',
}
UpperCamelCase_					:						List[Any]				= '''README.md'''
def 							A_     (__a				,   __a				,   __a				):

     '''simple docstring'''





     with open(__a				,   "r"				,   encoding="utf-8"				,   newline="\n"				) as f:
          A_				 =						f.read()
     A_ ,    A_				 =						REPLACE_PATTERNS[pattern]
     A_				 =						replace.replace("VERSION"				,   __a				)
     A_				 =						re_pattern.sub(__a				,   __a				)
     with open(__a				,   "w"				,   encoding="utf-8"				,   newline="\n"				) as f:
          f.write(__a				)
def 							A_     (__a				):

     '''simple docstring'''





     for folder, directories, fnames in os.walk(__a				):
          # Removing some of the folders with non-actively maintained examples from the walk
          if "research_projects" in directories:
               directories.remove("research_projects"				)
          if "legacy" in directories:
               directories.remove("legacy"				)
          for fname in fnames:
               if fname.endswith(".py"				):
                    update_version_in_file(os.path.join(__a				,   __a				)				,   __a				,   pattern="examples"				)
def 							A_     (__a				,   __a=False				):

     '''simple docstring'''





     for pattern, fname in REPLACE_FILES.items():
          update_version_in_file(__a				,   __a				,   __a				)
     if not patch:
          update_version_in_examples(__a				)
def 							A_     ():

     '''simple docstring'''





     A_				 =						"🤗 Transformers currently provides the following architectures"
     A_				 =						"1. Want to contribute a new model?"
     with open(__a				,   "r"				,   encoding="utf-8"				,   newline="\n"				) as f:
          A_				 =						f.readlines()

     # Find the start of the list.
     A_				 =						0
     while not lines[start_index].startswith(_start_prompt				):
          start_index += 1
     start_index += 1

     A_				 =						start_index
     # Update the lines in the model list.
     while not lines[index].startswith(_end_prompt				):
          if lines[index].startswith("1."				):
               A_				 =						lines[index].replace(
                   "https://huggingface.co/docs/transformers/main/model_doc"				,   "https://huggingface.co/docs/transformers/model_doc"				,   )
          index += 1

     with open(__a				,   "w"				,   encoding="utf-8"				,   newline="\n"				) as f:
          f.writelines(__a				)
def 							A_     ():

     '''simple docstring'''





     with open(REPLACE_FILES["init"]				,   "r"				) as f:
          A_				 =						f.read()
     A_				 =						REPLACE_PATTERNS["init"][0].search(__a				).groups()[0]
     return packaging.version.parse(__a				)
def 							A_     (__a=False				):

     '''simple docstring'''





     A_				 =						get_version()
     if patch and default_version.is_devrelease:
          raise ValueError("Can't create a patch version from the dev branch, checkout a released version!"				)
     if default_version.is_devrelease:
          A_				 =						default_version.base_version
     elif patch:
          A_				 =						f'{default_version.major}.{default_version.minor}.{default_version.micro + 1}'
     else:
          A_				 =						f'{default_version.major}.{default_version.minor + 1}.0'

     # Now let's ask nicely if that's the right one.
     A_				 =						input(f'Which version are you releasing? [{default_version}]'				)
     if len(__a				) == 0:
          A_				 =						default_version

     print(f'Updating version to {version}.'				)
     global_version_update(__a				,   patch=__a				)
     if not patch:
          print("Cleaning main README, don't forget to run `make fix-copies`."				)
          clean_main_ref_in_model_list()
def 							A_     ():

     '''simple docstring'''





     A_				 =						get_version()
     A_				 =						f'{current_version.major}.{current_version.minor + 1}.0.dev0'
     A_				 =						current_version.base_version

     # Check with the user we got that right.
     A_				 =						input(f'Which version are we developing now? [{dev_version}]'				)
     if len(__a				) == 0:
          A_				 =						dev_version

     print(f'Updating version to {version}.'				)
     global_version_update(__a				)
     print("Cleaning main README, don't forget to run `make fix-copies`."				)
     clean_main_ref_in_model_list()


if __name__ == "__main__":
  UpperCamelCase_					:						str				= argparse.ArgumentParser()
  parser.add_argument('''--post_release''', action='''store_true''', help='''Whether this is pre or post release.''')
  parser.add_argument('''--patch''', action='''store_true''', help='''Whether or not this is a patch release.''')
  UpperCamelCase_					:						Any				= parser.parse_args()
  if not args.post_release:
    pre_release_work(patch=args.patch)
  elif args.patch:
    print('''Nothing to do after a patch :-)''')
  else:
    post_release_work()






"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


UpperCamelCase_					:						Any				= logging.get_logger(__name__)

UpperCamelCase_					:						Any				= {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_					:						List[str]				= {
    '''vocab_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase_					:						Union[str, Any]				= {
    '''yjernite/retribert-base-uncased''': 512,
}


UpperCamelCase_					:						str				= {
    '''yjernite/retribert-base-uncased''': {'''do_lower_case''': True},
}



class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				VOCAB_FILES_NAMES
			snake_case	      =				PRETRAINED_VOCAB_FILES_MAP
			snake_case	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case	      =				PRETRAINED_INIT_CONFIGURATION
			snake_case	      =				RetriBertTokenizer
			snake_case	      =				["input_ids", "attention_mask"]




			def __init__(				self						:					Tuple       ,   _snake_case						:					int=None       ,   _snake_case						:					Any=None       ,   _snake_case						:					str=True       ,   _snake_case						:					List[Any]="[UNK]"       ,   _snake_case						:					Optional[Any]="[SEP]"       ,   _snake_case						:					str="[PAD]"       ,   _snake_case						:					List[str]="[CLS]"       ,   _snake_case						:					List[str]="[MASK]"       ,   _snake_case						:					Dict=True       ,   _snake_case						:					Optional[int]=None       ,   **_snake_case						:					Tuple       ,   )      -> Union[str, Any]:







								"""simple docstring"""
								super().__init__(
								    _snake_case       ,   tokenizer_file=_snake_case       ,   do_lower_case=_snake_case       ,   unk_token=_snake_case       ,   sep_token=_snake_case       ,   pad_token=_snake_case       ,   cls_token=_snake_case       ,   mask_token=_snake_case       ,   tokenize_chinese_chars=_snake_case       ,   strip_accents=_snake_case       ,   **_snake_case       ,   )

								A_				 =						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
								if (
								    normalizer_state.get("lowercase"       ,   _snake_case	) != do_lower_case
								    or normalizer_state.get("strip_accents"       ,   _snake_case	) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"       ,   _snake_case	) != tokenize_chinese_chars
								):
													A_				 =						getattr(_snake_case       ,   normalizer_state.pop("type"	)	)
													A_				 =						do_lower_case
													A_				 =						strip_accents
													A_				 =						tokenize_chinese_chars
													A_				 =						normalizer_class(**_snake_case	)

								A_				 =						do_lower_case




			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					List[Any]       ,   _snake_case						:					Union[str, Any]=None	)      -> List[Any]:







								"""simple docstring"""
								A_				 =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

								return output




			def      lowerCamelCase__     (				self						:					Any       ,   _snake_case						:					List[int]       ,   _snake_case						:					Optional[List[int]] = None	)      -> List[int]:







								"""simple docstring"""
								A_				 =						[self.sep_token_id]
								A_				 =						[self.cls_token_id]
								if token_ids_a is None:
													return len(cls + token_ids_a + sep	) * [0]
								return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					str       ,   _snake_case						:					Optional[str] = None	)      -> Tuple[str]:







								"""simple docstring"""
								A_				 =						self._tokenizer.model.save(_snake_case       ,   name=_snake_case	)
								return tuple(_snake_case	)




