






import collections
import os
import re
from pathlib import Path


UpperCamelCase    :      Optional[Any]  			=		'src/transformers'


# Matches is_xxx_available()
UpperCamelCase    :      int  			=		re.compile(r'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
UpperCamelCase    :      int  			=		re.compile(r'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
UpperCamelCase    :      List[str]  			=		re.compile(r'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
UpperCamelCase    :      str  			=		re.compile(r'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
UpperCamelCase    :      Optional[Any]  			=		re.compile(r'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
UpperCamelCase    :      Optional[Any]  			=		re.compile(r'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
UpperCamelCase    :      Optional[Any]  			=		re.compile(r'^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
UpperCamelCase    :      Any  			=		re.compile(r'^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
UpperCamelCase    :      List[str]  			=		re.compile(r'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
UpperCamelCase    :      Union[str, Any]  			=		re.compile(r'^\s*try:')
# Catches a line with else:
UpperCamelCase    :      List[Any]  			=		re.compile(r'^\s*else:')
def 						A__			(				__lowerCAmelCase		:			Optional[int]						):
						if _re_test_backend.search(__lowerCAmelCase						) is None:
												return None
						lowerCamelCase__         =  [b[0] for b in _re_backend.findall(__lowerCAmelCase						)]
						backends.sort()
						return "_and_".join(__lowerCAmelCase						)
def 						A__			(				__lowerCAmelCase		:			Tuple						):
						with open(__lowerCAmelCase   ,    """r"""   ,    encoding="""utf-8"""   ,    newline="""\n"""						) as f:
												lowerCamelCase__         =  f.readlines()

						lowerCamelCase__         =  0
						while line_index < len(__lowerCAmelCase						) and not lines[line_index].startswith("""_import_structure = {"""						):
												line_index += 1

						# If this is a traditional init, just return.
						if line_index >= len(__lowerCAmelCase						):
												return None

						# First grab the objects without a specific backend in _import_structure
						lowerCamelCase__         =  []
						while not lines[line_index].startswith("""if TYPE_CHECKING"""						) and find_backend(lines[line_index]						) is None:
												lowerCamelCase__         =  lines[line_index]
												# If we have everything on a single line, let's deal with it.
												if _re_one_line_import_struct.search(__lowerCAmelCase						):
																		lowerCamelCase__         =  _re_one_line_import_struct.search(__lowerCAmelCase						).groups()[0]
																		lowerCamelCase__         =  re.findall(R"""\[([^\]]+)\]"""   ,    __lowerCAmelCase						)
																		for imp in imports:
																								objects.extend([obj[1:-1] for obj in imp.split(""", """						)]						)
																		line_index += 1
																		continue
												lowerCamelCase__         =  _re_import_struct_key_value.search(__lowerCAmelCase						)
												if single_line_import_search is not None:
																		lowerCamelCase__         =  [obj[1:-1] for obj in single_line_import_search.groups()[0].split(""", """						) if len(__lowerCAmelCase						) > 0]
																		objects.extend(__lowerCAmelCase						)
												elif line.startswith(""" """ * 8 + """\""""						):
																		objects.append(line[9:-3]						)
												line_index += 1

						lowerCamelCase__         =  {"""none""": objects}
						# Let's continue with backend-specific objects in _import_structure
						while not lines[line_index].startswith("""if TYPE_CHECKING"""						):
												# If the line is an if not is_backend_available, we grab all objects associated.
												lowerCamelCase__         =  find_backend(lines[line_index]						)
												# Check if the backend declaration is inside a try block:
												if _re_try.search(lines[line_index - 1]						) is None:
																		lowerCamelCase__         =  None

												if backend is not None:
																		line_index += 1

																		# Scroll until we hit the else block of try-except-else
																		while _re_else.search(lines[line_index]						) is None:
																								line_index += 1

																		line_index += 1

																		lowerCamelCase__         =  []
																		# Until we unindent, add backend objects to the list
																		while len(lines[line_index]						) <= 1 or lines[line_index].startswith(""" """ * 4						):
																								lowerCamelCase__         =  lines[line_index]
																								if _re_import_struct_add_one.search(__lowerCAmelCase						) is not None:
																														objects.append(_re_import_struct_add_one.search(__lowerCAmelCase						).groups()[0]						)
																								elif _re_import_struct_add_many.search(__lowerCAmelCase						) is not None:
																														lowerCamelCase__         =  _re_import_struct_add_many.search(__lowerCAmelCase						).groups()[0].split(""", """						)
																														lowerCamelCase__         =  [obj[1:-1] for obj in imports if len(__lowerCAmelCase						) > 0]
																														objects.extend(__lowerCAmelCase						)
																								elif _re_between_brackets.search(__lowerCAmelCase						) is not None:
																														lowerCamelCase__         =  _re_between_brackets.search(__lowerCAmelCase						).groups()[0].split(""", """						)
																														lowerCamelCase__         =  [obj[1:-1] for obj in imports if len(__lowerCAmelCase						) > 0]
																														objects.extend(__lowerCAmelCase						)
																								elif _re_quote_object.search(__lowerCAmelCase						) is not None:
																														objects.append(_re_quote_object.search(__lowerCAmelCase						).groups()[0]						)
																								elif line.startswith(""" """ * 8 + """\""""						):
																														objects.append(line[9:-3]						)
																								elif line.startswith(""" """ * 12 + """\""""						):
																														objects.append(line[13:-3]						)
																								line_index += 1

																		lowerCamelCase__         =  objects
												else:
																		line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
						lowerCamelCase__         =  []
						while (
						    line_index < len(__lowerCAmelCase						)
						    and find_backend(lines[line_index]						) is None
						    and not lines[line_index].startswith("""else"""						)
						):
												lowerCamelCase__         =  lines[line_index]
												lowerCamelCase__         =  _re_import.search(__lowerCAmelCase						)
												if single_line_import_search is not None:
																		objects.extend(single_line_import_search.groups()[0].split(""", """						)						)
												elif line.startswith(""" """ * 8						):
																		objects.append(line[8:-2]						)
												line_index += 1

						lowerCamelCase__         =  {"""none""": objects}
						# Let's continue with backend-specific objects
						while line_index < len(__lowerCAmelCase						):
												# If the line is an if is_backend_available, we grab all objects associated.
												lowerCamelCase__         =  find_backend(lines[line_index]						)
												# Check if the backend declaration is inside a try block:
												if _re_try.search(lines[line_index - 1]						) is None:
																		lowerCamelCase__         =  None

												if backend is not None:
																		line_index += 1

																		# Scroll until we hit the else block of try-except-else
																		while _re_else.search(lines[line_index]						) is None:
																								line_index += 1

																		line_index += 1

																		lowerCamelCase__         =  []
																		# Until we unindent, add backend objects to the list
																		while len(lines[line_index]						) <= 1 or lines[line_index].startswith(""" """ * 8						):
																								lowerCamelCase__         =  lines[line_index]
																								lowerCamelCase__         =  _re_import.search(__lowerCAmelCase						)
																								if single_line_import_search is not None:
																														objects.extend(single_line_import_search.groups()[0].split(""", """						)						)
																								elif line.startswith(""" """ * 12						):
																														objects.append(line[12:-2]						)
																								line_index += 1

																		lowerCamelCase__         =  objects
												else:
																		line_index += 1

						return import_dict_objects, type_hint_objects
def 						A__			(				__lowerCAmelCase		:			Dict   ,    __lowerCAmelCase		:			str						):

						def find_duplicates(__lowerCAmelCase		:			Any						):
												return [k for k, v in collections.Counter(__lowerCAmelCase						).items() if v > 1]

						if list(import_dict_objects.keys()						) != list(type_hint_objects.keys()						):
												return ["Both sides of the init do not have the same backends!"]

						lowerCamelCase__         =  []
						for key in import_dict_objects.keys():
												lowerCamelCase__         =  find_duplicates(import_dict_objects[key]						)
												if duplicate_imports:
																		errors.append(F'''Duplicate _import_structure definitions for: {duplicate_imports}'''						)
												lowerCamelCase__         =  find_duplicates(type_hint_objects[key]						)
												if duplicate_type_hints:
																		errors.append(F'''Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'''						)

												if sorted(set(import_dict_objects[key]						)						) != sorted(set(type_hint_objects[key]						)						):
																		lowerCamelCase__         =  """base imports""" if key == """none""" else F'''{key} backend'''
																		errors.append(F'''Differences for {name}:'''						)
																		for a in type_hint_objects[key]:
																								if a not in import_dict_objects[key]:
																														errors.append(F'''  {a} in TYPE_HINT but not in _import_structure.'''						)
																		for a in import_dict_objects[key]:
																								if a not in type_hint_objects[key]:
																														errors.append(F'''  {a} in _import_structure but not in TYPE_HINT.'''						)
						return errors
def 						A__			(				):
						lowerCamelCase__         =  []
						for root, _, files in os.walk(__lowerCAmelCase						):
												if "__init__.py" in files:
																		lowerCamelCase__         =  os.path.join(__lowerCAmelCase   ,    """__init__.py"""						)
																		lowerCamelCase__         =  parse_init(__lowerCAmelCase						)
																		if objects is not None:
																								lowerCamelCase__         =  analyze_results(*__lowerCAmelCase						)
																								if len(__lowerCAmelCase						) > 0:
																														lowerCamelCase__         =  F'''Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'''
																														failures.append("""\n""".join(__lowerCAmelCase						)						)
						if len(__lowerCAmelCase						) > 0:
												raise ValueError("""\n\n""".join(__lowerCAmelCase						)						)
def 						A__			(				):
						lowerCamelCase__         =  []
						for path, directories, files in os.walk(__lowerCAmelCase						):
												for folder in directories:
																		# Ignore private modules
																		if folder.startswith("""_"""						):
																								directories.remove(__lowerCAmelCase						)
																								continue
																		# Ignore leftovers from branches (empty folders apart from pycache)
																		if len(list((Path(__lowerCAmelCase						) / folder).glob("""*.py"""						)						)						) == 0:
																								continue
																		lowerCamelCase__         =  str((Path(__lowerCAmelCase						) / folder).relative_to(__lowerCAmelCase						)						)
																		lowerCamelCase__         =  short_path.replace(os.path.sep   ,    """."""						)
																		submodules.append(__lowerCAmelCase						)
												for fname in files:
																		if fname == "__init__.py":
																								continue
																		lowerCamelCase__         =  str((Path(__lowerCAmelCase						) / fname).relative_to(__lowerCAmelCase						)						)
																		lowerCamelCase__         =  short_path.replace(""".py"""   ,    """"""						).replace(os.path.sep   ,    """."""						)
																		if len(submodule.split("""."""						)						) == 1:
																								submodules.append(__lowerCAmelCase						)
						return submodules


UpperCamelCase    :      Union[str, Any]  			=		[
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
    'models.esm.openfold_utils',
]
def 						A__			(				):
						# This is to make sure the transformers module imported is the one in the repo.
						from transformers.utils import direct_transformers_import

						lowerCamelCase__         =  direct_transformers_import(__lowerCAmelCase						)

						lowerCamelCase__         =  set(transformers._import_structure.keys()						)
						# This contains all the base keys of the _import_structure object defined in the init, but if the user is missing
						# some optional dependencies, they may not have all of them. Thus we read the init to read all additions and
						# (potentiall re-) add them.
						with open(os.path.join(__lowerCAmelCase   ,    """__init__.py"""						)   ,    """r"""						) as f:
												lowerCamelCase__         =  f.read()
						import_structure_keys.update(set(re.findall(R"""import_structure\[\"([^\"]*)\"\]"""   ,    __lowerCAmelCase						)						)						)

						lowerCamelCase__         =  [
						    module
						    for module in get_transformers_submodules()
						    if module not in IGNORE_SUBMODULES and module not in import_structure_keys
						]

						if len(__lowerCAmelCase						) > 0:
												lowerCamelCase__         =  """\n""".join(F'''- {module}''' for module in module_not_registered						)
												raise ValueError(
												    """The following submodules are not properly registed in the main init of Transformers:\n"""
												    F'''{list_of_modules}\n'''
												    """Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."""						)


if __name__ == "__main__":
							check_all_inits()
							check_submodules()









'''simple docstring'''







import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing




class 		UpperCamelCase__    (a			):




		'''simple docstring'''





		def __init__(      self   ,_lowerCAmelCase = "▁"   ,_lowerCAmelCase = True   ,_lowerCAmelCase = "<unk>"   ,_lowerCAmelCase = "</s>"   ,_lowerCAmelCase = "<pad>"   ,):
								lowerCamelCase__         =  {
								    """pad""": {"""id""": 0, """token""": pad_token},
								    """eos""": {"""id""": 1, """token""": eos_token},
								    """unk""": {"""id""": 2, """token""": unk_token},
								}

								lowerCamelCase__         =  [None] * len(self.special_tokens       )
								for token_dict in self.special_tokens.values():
														lowerCamelCase__         =  token_dict["""token"""]

								lowerCamelCase__         =  Tokenizer(Unigram()       )

								lowerCamelCase__         =  normalizers.Sequence(
								    [
								        normalizers.Nmt(),
								        normalizers.NFKC(),
								        normalizers.Replace(Regex(""" {2,}"""       )   ,""" """       ),
								        normalizers.Lowercase(),
								    ]       )
								lowerCamelCase__         =  pre_tokenizers.Sequence(
								    [
								        pre_tokenizers.Metaspace(replacement=_lowerCAmelCase   ,add_prefix_space=_lowerCAmelCase       ),
								        pre_tokenizers.Digits(individual_digits=_lowerCAmelCase       ),
								        pre_tokenizers.Punctuation(),
								    ]       )
								lowerCamelCase__         =  decoders.Metaspace(replacement=_lowerCAmelCase   ,add_prefix_space=_lowerCAmelCase       )

								lowerCamelCase__         =  TemplateProcessing(
								    single=F'''$A {self.special_tokens["eos"]["token"]}'''   ,special_tokens=[(self.special_tokens["""eos"""]["""token"""], self.special_tokens["""eos"""]["""id"""])]   ,)

								lowerCamelCase__         =  {
								    """model""": """SentencePieceUnigram""",
								    """replacement""": replacement,
								    """add_prefix_space""": add_prefix_space,
								}

								super().__init__(_lowerCAmelCase   ,_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase = 80_00   ,_lowerCAmelCase = True   ,):
								lowerCamelCase__         =  trainers.UnigramTrainer(
								    vocab_size=_lowerCAmelCase   ,special_tokens=self.special_tokens_list   ,show_progress=_lowerCAmelCase   ,)

								if isinstance(_lowerCAmelCase   ,_lowerCAmelCase       ):
														lowerCamelCase__         =  [files]
								self._tokenizer.train(_lowerCAmelCase   ,trainer=_lowerCAmelCase       )

								self.add_unk_id()






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase = 80_00   ,_lowerCAmelCase = True   ,):
								lowerCamelCase__         =  trainers.UnigramTrainer(
								    vocab_size=_lowerCAmelCase   ,special_tokens=self.special_tokens_list   ,show_progress=_lowerCAmelCase   ,)

								self._tokenizer.train_from_iterator(_lowerCAmelCase   ,trainer=_lowerCAmelCase       )

								self.add_unk_id()






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  json.loads(self._tokenizer.to_str()       )

								lowerCamelCase__         =  self.special_tokens["""unk"""]["""id"""]

								lowerCamelCase__         =  Tokenizer.from_str(json.dumps(_lowerCAmelCase       )       )

