






"""simple docstring"""
import collections
import importlib.util
import os
import re
from pathlib import Path


lowerCamelCase_       			=   '''src/transformers'''


# Matches is_xxx_available()
lowerCamelCase_       			=   re.compile(r'''is\_([a-z_]*)_available()''')
# Catches a one-line _import_struct = {xxx}
lowerCamelCase_       			=   re.compile(r'''^_import_structure\s+=\s+\{([^\}]+)\}''')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
lowerCamelCase_       			=   re.compile(r'''\s+"\S*":\s+\[([^\]]*)\]''')
# Catches a line if not is_foo_available
lowerCamelCase_       			=   re.compile(r'''^\s*if\s+not\s+is\_[a-z_]*\_available\(\)''')
# Catches a line _import_struct["bla"].append("foo")
lowerCamelCase_       			=   re.compile(r'''^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)''')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
lowerCamelCase_       			=   re.compile(r'''^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]''')
# Catches a line with an object between quotes and a comma:     "MyModel",
lowerCamelCase_       			=   re.compile('''^\s+"([^"]+)",''')
# Catches a line with objects between brackets only:    ["foo", "bar"],
lowerCamelCase_       			=   re.compile('''^\s+\[([^\]]+)\]''')
# Catches a line with from foo import bar, bla, boo
lowerCamelCase_       			=   re.compile(r'''\s+from\s+\S*\s+import\s+([^\(\s].*)\n''')
# Catches a line with try:
lowerCamelCase_       			=   re.compile(r'''^\s*try:''')
# Catches a line with else:
lowerCamelCase_       			=   re.compile(r'''^\s*else:''')

def        snake_case   (     A__		):
			if _re_test_backend.search(A__		) is None:
						return None
			UpperCAmelCase_  :				List[str]      =      [b[0] for b in _re_backend.findall(A__		)]
			backends.sort()
			return "_and_".join(A__		)

def        snake_case   (     A__		):
			with open(A__			,"r"			,encoding="utf-8"			,newline="\n"		) as f:
						UpperCAmelCase_  :				Dict      =      f.readlines()

			UpperCAmelCase_  :				List[str]      =      0
			while line_index < len(A__		) and not lines[line_index].startswith("_import_structure = {"		):
						line_index += 1

			# If this is a traditional init, just return.
			if line_index >= len(A__		):
						return None

			# First grab the objects without a specific backend in _import_structure
			UpperCAmelCase_  :				Dict      =      []
			while not lines[line_index].startswith("if TYPE_CHECKING"		) and find_backend(lines[line_index]		) is None:
						UpperCAmelCase_  :				Optional[Any]      =      lines[line_index]
						# If we have everything on a single line, let's deal with it.
						if _re_one_line_import_struct.search(A__		):
									UpperCAmelCase_  :				Optional[int]      =      _re_one_line_import_struct.search(A__		).groups()[0]
									UpperCAmelCase_  :				str      =      re.findall("\[([^\]]+)\]"			,A__		)
									for imp in imports:
												objects.extend([obj[1:-1] for obj in imp.split(", "		)]		)
									line_index += 1
									continue
						UpperCAmelCase_  :				Union[str, Any]      =      _re_import_struct_key_value.search(A__		)
						if single_line_import_search is not None:
									UpperCAmelCase_  :				Optional[int]      =      [obj[1:-1] for obj in single_line_import_search.groups()[0].split(", "		) if len(A__		) > 0]
									objects.extend(A__		)
						elif line.startswith(" " * 8 + "\""		):
									objects.append(line[9:-3]		)
						line_index += 1

			UpperCAmelCase_  :				Any      =      {"none": objects}
			# Let's continue with backend-specific objects in _import_structure
			while not lines[line_index].startswith("if TYPE_CHECKING"		):
						# If the line is an if not is_backend_available, we grab all objects associated.
						UpperCAmelCase_  :				List[str]      =      find_backend(lines[line_index]		)
						# Check if the backend declaration is inside a try block:
						if _re_try.search(lines[line_index - 1]		) is None:
									UpperCAmelCase_  :				Tuple      =      None

						if backend is not None:
									line_index += 1

									# Scroll until we hit the else block of try-except-else
									while _re_else.search(lines[line_index]		) is None:
												line_index += 1

									line_index += 1

									UpperCAmelCase_  :				int      =      []
									# Until we unindent, add backend objects to the list
									while len(lines[line_index]		) <= 1 or lines[line_index].startswith(" " * 4		):
												UpperCAmelCase_  :				List[Any]      =      lines[line_index]
												if _re_import_struct_add_one.search(A__		) is not None:
															objects.append(_re_import_struct_add_one.search(A__		).groups()[0]		)
												elif _re_import_struct_add_many.search(A__		) is not None:
															UpperCAmelCase_  :				Any      =      _re_import_struct_add_many.search(A__		).groups()[0].split(", "		)
															UpperCAmelCase_  :				int      =      [obj[1:-1] for obj in imports if len(A__		) > 0]
															objects.extend(A__		)
												elif _re_between_brackets.search(A__		) is not None:
															UpperCAmelCase_  :				int      =      _re_between_brackets.search(A__		).groups()[0].split(", "		)
															UpperCAmelCase_  :				Optional[int]      =      [obj[1:-1] for obj in imports if len(A__		) > 0]
															objects.extend(A__		)
												elif _re_quote_object.search(A__		) is not None:
															objects.append(_re_quote_object.search(A__		).groups()[0]		)
												elif line.startswith(" " * 8 + "\""		):
															objects.append(line[9:-3]		)
												elif line.startswith(" " * 12 + "\""		):
															objects.append(line[13:-3]		)
												line_index += 1

									UpperCAmelCase_  :				int      =      objects
						else:
									line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
			UpperCAmelCase_  :				int      =      []
			while (
			    line_index < len(A__		)
			    and find_backend(lines[line_index]		) is None
			    and not lines[line_index].startswith("else"		)
			):
						UpperCAmelCase_  :				int      =      lines[line_index]
						UpperCAmelCase_  :				Union[str, Any]      =      _re_import.search(A__		)
						if single_line_import_search is not None:
									objects.extend(single_line_import_search.groups()[0].split(", "		)		)
						elif line.startswith(" " * 8		):
									objects.append(line[8:-2]		)
						line_index += 1

			UpperCAmelCase_  :				Optional[Any]      =      {"none": objects}
			# Let's continue with backend-specific objects
			while line_index < len(A__		):
						# If the line is an if is_backend_available, we grab all objects associated.
						UpperCAmelCase_  :				Optional[int]      =      find_backend(lines[line_index]		)
						# Check if the backend declaration is inside a try block:
						if _re_try.search(lines[line_index - 1]		) is None:
									UpperCAmelCase_  :				Any      =      None

						if backend is not None:
									line_index += 1

									# Scroll until we hit the else block of try-except-else
									while _re_else.search(lines[line_index]		) is None:
												line_index += 1

									line_index += 1

									UpperCAmelCase_  :				Union[str, Any]      =      []
									# Until we unindent, add backend objects to the list
									while len(lines[line_index]		) <= 1 or lines[line_index].startswith(" " * 8		):
												UpperCAmelCase_  :				Optional[Any]      =      lines[line_index]
												UpperCAmelCase_  :				List[Any]      =      _re_import.search(A__		)
												if single_line_import_search is not None:
															objects.extend(single_line_import_search.groups()[0].split(", "		)		)
												elif line.startswith(" " * 12		):
															objects.append(line[12:-2]		)
												line_index += 1

									UpperCAmelCase_  :				str      =      objects
						else:
									line_index += 1

			return import_dict_objects, type_hint_objects

def        snake_case   (     A__			,A__		):

			def find_duplicates(A__		):
						return [k for k, v in collections.Counter(A__		).items() if v > 1]

			if list(import_dict_objects.keys()		) != list(type_hint_objects.keys()		):
						return ["Both sides of the init do not have the same backends!"]

			UpperCAmelCase_  :				Optional[Any]      =      []
			for key in import_dict_objects.keys():
						UpperCAmelCase_  :				str      =      find_duplicates(import_dict_objects[key]		)
						if duplicate_imports:
									errors.append(F"""Duplicate _import_structure definitions for: {duplicate_imports}"""		)
						UpperCAmelCase_  :				int      =      find_duplicates(type_hint_objects[key]		)
						if duplicate_type_hints:
									errors.append(F"""Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"""		)

						if sorted(set(import_dict_objects[key]		)		) != sorted(set(type_hint_objects[key]		)		):
									UpperCAmelCase_  :				Tuple      =      "base imports" if key == "none" else F"""{key} backend"""
									errors.append(F"""Differences for {name}:"""		)
									for a in type_hint_objects[key]:
												if a not in import_dict_objects[key]:
															errors.append(F"""  {a} in TYPE_HINT but not in _import_structure."""		)
									for a in import_dict_objects[key]:
												if a not in type_hint_objects[key]:
															errors.append(F"""  {a} in _import_structure but not in TYPE_HINT."""		)
			return errors

def        snake_case   (     ):
			UpperCAmelCase_  :				Union[str, Any]      =      []
			for root, _, files in os.walk(A__		):
						if "__init__.py" in files:
									UpperCAmelCase_  :				str      =      os.path.join(A__			,"__init__.py"		)
									UpperCAmelCase_  :				Any      =      parse_init(A__		)
									if objects is not None:
												UpperCAmelCase_  :				int      =      analyze_results(*A__		)
												if len(A__		) > 0:
															UpperCAmelCase_  :				Union[str, Any]      =      F"""Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"""
															failures.append("\n".join(A__		)		)
			if len(A__		) > 0:
						raise ValueError("\n\n".join(A__		)		)

def        snake_case   (     ):
			UpperCAmelCase_  :				List[str]      =      []
			for path, directories, files in os.walk(A__		):
						for folder in directories:
									# Ignore private modules
									if folder.startswith("_"		):
												directories.remove(A__		)
												continue
									# Ignore leftovers from branches (empty folders apart from pycache)
									if len(list((Path(A__		) / folder).glob("*.py"		)		)		) == 0:
												continue
									UpperCAmelCase_  :				int      =      str((Path(A__		) / folder).relative_to(A__		)		)
									UpperCAmelCase_  :				List[Any]      =      short_path.replace(os.path.sep			,"."		)
									submodules.append(A__		)
						for fname in files:
									if fname == "__init__.py":
												continue
									UpperCAmelCase_  :				str      =      str((Path(A__		) / fname).relative_to(A__		)		)
									UpperCAmelCase_  :				Tuple      =      short_path.replace(".py"			,""		).replace(os.path.sep			,"."		)
									if len(submodule.split("."		)		) == 1:
												submodules.append(A__		)
			return submodules


lowerCamelCase_       			=   [
    '''convert_pytorch_checkpoint_to_tf2''',
    '''modeling_flax_pytorch_utils''',
]

def        snake_case   (     ):
			# This is to make sure the transformers module imported is the one in the repo.
			UpperCAmelCase_  :				Dict      =      importlib.util.spec_from_file_location(
			    "transformers"			,os.path.join(A__			,"__init__.py"		)			,submodule_search_locations=[PATH_TO_TRANSFORMERS]			,)
			UpperCAmelCase_  :				Optional[Any]      =      spec.loader.load_module()

			UpperCAmelCase_  :				Optional[int]      =      [
			    module
			    for module in get_transformers_submodules()
			    if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
			]
			if len(A__		) > 0:
						UpperCAmelCase_  :				Dict      =      "\n".join(F"""- {module}""" for module in module_not_registered		)
						raise ValueError(
						    "The following submodules are not properly registered in the main init of Transformers:\n"
						    F"""{list_of_modules}\n"""
						    "Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."		)


if __name__ == "__main__":
			check_all_inits()
			check_submodules()











"""simple docstring"""
import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


lowerCamelCase_       			=   logging.getLogger()







@unittest.skip('''Temporarily disable the doc tests.'''  )
@require_torch
@require_tf
@slow
class     UpperCamelCase_ (unittest.TestCase  ):





	def   _SCREAMING_SNAKE_CASE   (   self      :				Union[str, Any]			,			lowerCAmelCase_      :				Path			,			lowerCAmelCase_      :				Union[str, None] = None			,			lowerCAmelCase_      :				Union[List[str], None] = None			,			lowerCAmelCase_      :				Union[str, List[str], None] = None			,			lowerCAmelCase_      :				bool = True			,			)		->    Tuple:
				UpperCAmelCase_  :				Optional[Any]      =      [file for file in os.listdir(lowerCAmelCase_		) if os.path.isfile(os.path.join(lowerCAmelCase_			,			lowerCAmelCase_		)		)]

				if identifier is not None:
							UpperCAmelCase_  :				Optional[int]      =      [file for file in files if identifier in file]

				if n_identifier is not None:
							if isinstance(lowerCAmelCase_			,			lowerCAmelCase_		):
										for n_ in n_identifier:
													UpperCAmelCase_  :				List[Any]      =      [file for file in files if n_ not in file]
							else:
										UpperCAmelCase_  :				Optional[Any]      =      [file for file in files if n_identifier not in file]

				UpperCAmelCase_  :				List[Any]      =      ignore_files or []
				ignore_files.append("__init__.py"		)
				UpperCAmelCase_  :				Optional[Any]      =      [file for file in files if file not in ignore_files]

				for file in files:
							# Open all files
							print("Testing"			,			lowerCAmelCase_		)

							if only_modules:
										UpperCAmelCase_  :				str      =      file.split("."		)[0]
										try:
													UpperCAmelCase_  :				Tuple      =      getattr(lowerCAmelCase_			,			lowerCAmelCase_		)
													UpperCAmelCase_  :				int      =      doctest.DocTestSuite(lowerCAmelCase_		)
													UpperCAmelCase_  :				str      =      unittest.TextTestRunner().run(lowerCAmelCase_		)
													self.assertIs(len(result.failures		)			,			0		)
										except AttributeError:
													logger.info(f"""{module_identifier} is not a module."""		)
							else:
										UpperCAmelCase_  :				List[Any]      =      doctest.testfile(str(".." / directory / file		)			,			optionflags=doctest.ELLIPSIS		)
										self.assertIs(result.failed			,			0		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[int]		)		->    List[Any]:
				UpperCAmelCase_  :				Optional[int]      =      Path("src/transformers"		)
				UpperCAmelCase_  :				List[Any]      =      "modeling"
				UpperCAmelCase_  :				Tuple      =      [
				    "modeling_ctrl.py",
				    "modeling_tf_ctrl.py",
				]
				self.analyze_directory(lowerCAmelCase_			,			identifier=lowerCAmelCase_			,			ignore_files=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Union[str, Any]		)		->    List[Any]:
				UpperCAmelCase_  :				List[str]      =      Path("src/transformers"		)
				UpperCAmelCase_  :				int      =      "tokenization"
				self.analyze_directory(lowerCAmelCase_			,			identifier=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[Any]		)		->    Optional[Any]:
				UpperCAmelCase_  :				int      =      Path("src/transformers"		)
				UpperCAmelCase_  :				Any      =      "configuration"
				self.analyze_directory(lowerCAmelCase_			,			identifier=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Any		)		->    Any:
				UpperCAmelCase_  :				int      =      Path("src/transformers"		)
				UpperCAmelCase_  :				List[Any]      =      ["configuration", "modeling", "tokenization"]
				self.analyze_directory(lowerCAmelCase_			,			n_identifier=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				List[str]		)		->    Dict:
				UpperCAmelCase_  :				List[str]      =      Path("docs/source"		)
				UpperCAmelCase_  :				int      =      ["favicon.ico"]
				self.analyze_directory(lowerCAmelCase_			,			ignore_files=lowerCAmelCase_			,			only_modules=lowerCAmelCase_		)



