



"""simple docstring"""

import collections
import os
import re
from pathlib import Path


a_			   =					'src/transformers'


# Matches is_xxx_available()
a_			   =					re.compile(r'''is\_([a-z_]*)_available()''')
# Catches a one-line _import_struct = {xxx}
a_			   =					re.compile(r'''^_import_structure\s+=\s+\{([^\}]+)\}''')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
a_			   =					re.compile(r'''\s+"\S*":\s+\[([^\]]*)\]''')
# Catches a line if not is_foo_available
a_			   =					re.compile(r'''^\s*if\s+not\s+is\_[a-z_]*\_available\(\)''')
# Catches a line _import_struct["bla"].append("foo")
a_			   =					re.compile(r'''^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)''')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
a_			   =					re.compile(r'''^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]''')
# Catches a line with an object between quotes and a comma:     "MyModel",
a_			   =					re.compile(r'''^\s+"([^"]+)",''')
# Catches a line with objects between brackets only:    ["foo", "bar"],
a_			   =					re.compile(r'''^\s+\[([^\]]+)\]''')
# Catches a line with from foo import bar, bla, boo
a_			   =					re.compile(r'''\s+from\s+\S*\s+import\s+([^\(\s].*)\n''')
# Catches a line with try:
a_			   =					re.compile(r'''^\s*try:''')
# Catches a line with else:
a_			   =					re.compile(r'''^\s*else:''')







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Union[str, Any]			):



					"""simple docstring"""





					if _re_test_backend.search(_SCREAMING_SNAKE_CASE			) is None:
										return None
					snake_case_					:		List[str] 					=					[b[0] for b in _re_backend.findall(_SCREAMING_SNAKE_CASE			)]
					backends.sort()
					return "_and_".join(_SCREAMING_SNAKE_CASE			)







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  List[str]			):



					"""simple docstring"""





					with open(_SCREAMING_SNAKE_CASE				,    """r"""				,    encoding="""utf-8"""				,    newline="""\n"""			) as f:
										snake_case_					:		str 					=					f.readlines()

					snake_case_					:		str 					=					0
					while line_index < len(_SCREAMING_SNAKE_CASE			) and not lines[line_index].startswith("""_import_structure = {"""			):
										line_index += 1

					# If this is a traditional init, just return.
					if line_index >= len(_SCREAMING_SNAKE_CASE			):
										return None

					# First grab the objects without a specific backend in _import_structure
					snake_case_					:		List[Any] 					=					[]
					while not lines[line_index].startswith("""if TYPE_CHECKING"""			) and find_backend(lines[line_index]			) is None:
										snake_case_					:		int 					=					lines[line_index]
										# If we have everything on a single line, let's deal with it.
										if _re_one_line_import_struct.search(_SCREAMING_SNAKE_CASE			):
															snake_case_					:		Union[str, Any] 					=					_re_one_line_import_struct.search(_SCREAMING_SNAKE_CASE			).groups()[0]
															snake_case_					:		List[str] 					=					re.findall(R"""\[([^\]]+)\]"""				,    _SCREAMING_SNAKE_CASE			)
															for imp in imports:
																				objects.extend([obj[1:-1] for obj in imp.split(""", """			)]			)
															line_index += 1
															continue
										snake_case_					:		int 					=					_re_import_struct_key_value.search(_SCREAMING_SNAKE_CASE			)
										if single_line_import_search is not None:
															snake_case_					:		Union[str, Any] 					=					[obj[1:-1] for obj in single_line_import_search.groups()[0].split(""", """			) if len(_SCREAMING_SNAKE_CASE			) > 0]
															objects.extend(_SCREAMING_SNAKE_CASE			)
										elif line.startswith(""" """ * 8 + """\""""			):
															objects.append(line[9:-3]			)
										line_index += 1

					snake_case_					:		int 					=					{"""none""": objects}
					# Let's continue with backend-specific objects in _import_structure
					while not lines[line_index].startswith("""if TYPE_CHECKING"""			):
										# If the line is an if not is_backend_available, we grab all objects associated.
										snake_case_					:		Dict 					=					find_backend(lines[line_index]			)
										# Check if the backend declaration is inside a try block:
										if _re_try.search(lines[line_index - 1]			) is None:
															snake_case_					:		Any 					=					None

										if backend is not None:
															line_index += 1

															# Scroll until we hit the else block of try-except-else
															while _re_else.search(lines[line_index]			) is None:
																				line_index += 1

															line_index += 1

															snake_case_					:		List[Any] 					=					[]
															# Until we unindent, add backend objects to the list
															while len(lines[line_index]			) <= 1 or lines[line_index].startswith(""" """ * 4			):
																				snake_case_					:		int 					=					lines[line_index]
																				if _re_import_struct_add_one.search(_SCREAMING_SNAKE_CASE			) is not None:
																									objects.append(_re_import_struct_add_one.search(_SCREAMING_SNAKE_CASE			).groups()[0]			)
																				elif _re_import_struct_add_many.search(_SCREAMING_SNAKE_CASE			) is not None:
																									snake_case_					:		str 					=					_re_import_struct_add_many.search(_SCREAMING_SNAKE_CASE			).groups()[0].split(""", """			)
																									snake_case_					:		Dict 					=					[obj[1:-1] for obj in imports if len(_SCREAMING_SNAKE_CASE			) > 0]
																									objects.extend(_SCREAMING_SNAKE_CASE			)
																				elif _re_between_brackets.search(_SCREAMING_SNAKE_CASE			) is not None:
																									snake_case_					:		Any 					=					_re_between_brackets.search(_SCREAMING_SNAKE_CASE			).groups()[0].split(""", """			)
																									snake_case_					:		int 					=					[obj[1:-1] for obj in imports if len(_SCREAMING_SNAKE_CASE			) > 0]
																									objects.extend(_SCREAMING_SNAKE_CASE			)
																				elif _re_quote_object.search(_SCREAMING_SNAKE_CASE			) is not None:
																									objects.append(_re_quote_object.search(_SCREAMING_SNAKE_CASE			).groups()[0]			)
																				elif line.startswith(""" """ * 8 + """\""""			):
																									objects.append(line[9:-3]			)
																				elif line.startswith(""" """ * 1_2 + """\""""			):
																									objects.append(line[1_3:-3]			)
																				line_index += 1

															snake_case_					:		str 					=					objects
										else:
															line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
					snake_case_					:		Dict 					=					[]
					while (
					    line_index < len(_SCREAMING_SNAKE_CASE			)
					    and find_backend(lines[line_index]			) is None
					    and not lines[line_index].startswith("""else"""			)
					):
										snake_case_					:		List[str] 					=					lines[line_index]
										snake_case_					:		str 					=					_re_import.search(_SCREAMING_SNAKE_CASE			)
										if single_line_import_search is not None:
															objects.extend(single_line_import_search.groups()[0].split(""", """			)			)
										elif line.startswith(""" """ * 8			):
															objects.append(line[8:-2]			)
										line_index += 1

					snake_case_					:		List[str] 					=					{"""none""": objects}
					# Let's continue with backend-specific objects
					while line_index < len(_SCREAMING_SNAKE_CASE			):
										# If the line is an if is_backend_available, we grab all objects associated.
										snake_case_					:		Dict 					=					find_backend(lines[line_index]			)
										# Check if the backend declaration is inside a try block:
										if _re_try.search(lines[line_index - 1]			) is None:
															snake_case_					:		List[Any] 					=					None

										if backend is not None:
															line_index += 1

															# Scroll until we hit the else block of try-except-else
															while _re_else.search(lines[line_index]			) is None:
																				line_index += 1

															line_index += 1

															snake_case_					:		Optional[int] 					=					[]
															# Until we unindent, add backend objects to the list
															while len(lines[line_index]			) <= 1 or lines[line_index].startswith(""" """ * 8			):
																				snake_case_					:		List[str] 					=					lines[line_index]
																				snake_case_					:		List[Any] 					=					_re_import.search(_SCREAMING_SNAKE_CASE			)
																				if single_line_import_search is not None:
																									objects.extend(single_line_import_search.groups()[0].split(""", """			)			)
																				elif line.startswith(""" """ * 1_2			):
																									objects.append(line[1_2:-2]			)
																				line_index += 1

															snake_case_					:		Union[str, Any] 					=					objects
										else:
															line_index += 1

					return import_dict_objects, type_hint_objects







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Tuple				,    SCREAMING_SNAKE_CASE__ :  Optional[Any]			):



					"""simple docstring"""





					def find_duplicates(SCREAMING_SNAKE_CASE__ :  Dict			):
										return [k for k, v in collections.Counter(_SCREAMING_SNAKE_CASE			).items() if v > 1]

					if list(import_dict_objects.keys()			) != list(type_hint_objects.keys()			):
										return ["Both sides of the init do not have the same backends!"]

					snake_case_					:		Dict 					=					[]
					for key in import_dict_objects.keys():
										snake_case_					:		Tuple 					=					find_duplicates(import_dict_objects[key]			)
										if duplicate_imports:
															errors.append(f'Duplicate _import_structure definitions for: {duplicate_imports}'			)
										snake_case_					:		Tuple 					=					find_duplicates(type_hint_objects[key]			)
										if duplicate_type_hints:
															errors.append(f'Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'			)

										if sorted(set(import_dict_objects[key]			)			) != sorted(set(type_hint_objects[key]			)			):
															snake_case_					:		str 					=					"""base imports""" if key == """none""" else f'{key} backend'
															errors.append(f'Differences for {name}:'			)
															for a in type_hint_objects[key]:
																				if a not in import_dict_objects[key]:
																									errors.append(f'  {a} in TYPE_HINT but not in _import_structure.'			)
															for a in import_dict_objects[key]:
																				if a not in type_hint_objects[key]:
																									errors.append(f'  {a} in _import_structure but not in TYPE_HINT.'			)
					return errors







def 							SCREAMING_SNAKE_CASE__      (		):



					"""simple docstring"""





					snake_case_					:		Tuple 					=					[]
					for root, _, files in os.walk(_SCREAMING_SNAKE_CASE			):
										if "__init__.py" in files:
															snake_case_					:		List[Any] 					=					os.path.join(_SCREAMING_SNAKE_CASE				,    """__init__.py"""			)
															snake_case_					:		str 					=					parse_init(_SCREAMING_SNAKE_CASE			)
															if objects is not None:
																				snake_case_					:		Optional[Any] 					=					analyze_results(*_SCREAMING_SNAKE_CASE			)
																				if len(_SCREAMING_SNAKE_CASE			) > 0:
																									snake_case_					:		str 					=					f'Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'
																									failures.append("""\n""".join(_SCREAMING_SNAKE_CASE			)			)
					if len(_SCREAMING_SNAKE_CASE			) > 0:
										raise ValueError("""\n\n""".join(_SCREAMING_SNAKE_CASE			)			)







def 							SCREAMING_SNAKE_CASE__      (		):



					"""simple docstring"""





					snake_case_					:		int 					=					[]
					for path, directories, files in os.walk(_SCREAMING_SNAKE_CASE			):
										for folder in directories:
															# Ignore private modules
															if folder.startswith("""_"""			):
																				directories.remove(_SCREAMING_SNAKE_CASE			)
																				continue
															# Ignore leftovers from branches (empty folders apart from pycache)
															if len(list((Path(_SCREAMING_SNAKE_CASE			) / folder).glob("""*.py"""			)			)			) == 0:
																				continue
															snake_case_					:		int 					=					str((Path(_SCREAMING_SNAKE_CASE			) / folder).relative_to(_SCREAMING_SNAKE_CASE			)			)
															snake_case_					:		Union[str, Any] 					=					short_path.replace(os.path.sep				,    """."""			)
															submodules.append(_SCREAMING_SNAKE_CASE			)
										for fname in files:
															if fname == "__init__.py":
																				continue
															snake_case_					:		Tuple 					=					str((Path(_SCREAMING_SNAKE_CASE			) / fname).relative_to(_SCREAMING_SNAKE_CASE			)			)
															snake_case_					:		List[str] 					=					short_path.replace(""".py"""				,    """"""			).replace(os.path.sep				,    """."""			)
															if len(submodule.split("""."""			)			) == 1:
																				submodules.append(_SCREAMING_SNAKE_CASE			)
					return submodules


a_			   =					[
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
    'models.esm.openfold_utils',
]







def 							SCREAMING_SNAKE_CASE__      (		):



					"""simple docstring"""





					from transformers.utils import direct_transformers_import

					snake_case_					:		Optional[int] 					=					direct_transformers_import(_SCREAMING_SNAKE_CASE			)

					snake_case_					:		List[str] 					=					set(transformers._import_structure.keys()			)
					# This contains all the base keys of the _import_structure object defined in the init, but if the user is missing
					# some optional dependencies, they may not have all of them. Thus we read the init to read all additions and
					# (potentiall re-) add them.
					with open(os.path.join(_SCREAMING_SNAKE_CASE				,    """__init__.py"""			)				,    """r"""			) as f:
										snake_case_					:		Any 					=					f.read()
					import_structure_keys.update(set(re.findall(R"""import_structure\[\"([^\"]*)\"\]"""				,    _SCREAMING_SNAKE_CASE			)			)			)

					snake_case_					:		List[Any] 					=					[
					    module
					    for module in get_transformers_submodules()
					    if module not in IGNORE_SUBMODULES and module not in import_structure_keys
					]

					if len(_SCREAMING_SNAKE_CASE			) > 0:
										snake_case_					:		str 					=					"""\n""".join(f'- {module}' for module in module_not_registered			)
										raise ValueError(
										    """The following submodules are not properly registed in the main init of Transformers:\n"""
										    f'{list_of_modules}\n'
										    """Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."""			)


if __name__ == "__main__":
			check_all_inits()
			check_submodules()









"""simple docstring"""

import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
a_			   =					logging.getLogger(__name__)

if __name__ == "__main__":
			a_			   =					argparse.ArgumentParser(
			    description='''Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'''
			)
			parser.add_argument(
			    '''--data_file''', type=str, default='''data/dump.bert-base-uncased.pickle''', help='''The binarized dataset.'''
			)
			parser.add_argument(
			    '''--token_counts_dump''', type=str, default='''data/token_counts.bert-base-uncased.pickle''', help='''The dump file.'''
			)
			parser.add_argument('''--vocab_size''', default=30522, type=int)
			a_			   =					parser.parse_args()

			logger.info(F'''Loading data from {args.data_file}''')
			with open(args.data_file, '''rb''') as fp:
						a_			   =					pickle.load(fp)

			logger.info('''Counting occurrences for MLM.''')
			a_			   =					Counter()
			for tk_ids in data:
						counter.update(tk_ids)
			a_			   =					[0] * args.vocab_size
			for k, v in counter.items():
						a_			   =					v

			logger.info(F'''Dump to {args.token_counts_dump}''')
			with open(args.token_counts_dump, '''wb''') as handle:
						pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)




