



import argparse
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_dummies.py
__snake_case		:Tuple   	=  '''src/diffusers'''

# Matches is_xxx_available()
__snake_case		:Optional[Any]   	=  re.compile(r'''is\_([a-z_]*)_available\(\)''')
# Matches from xxx import bla
__snake_case		:int   	=  re.compile(r'''\s+from\s+\S*\s+import\s+([^\(\s].*)\n''')


__snake_case		:int   	=  '''
{0} = None
'''

__snake_case		:Any   	=  '''
class {0}(metaclass=DummyObject):
    _backends = {1}

    def __init__(self, *args, **kwargs):
        requires_backends(self, {1})

    @classmethod
    def from_config(cls, *args, **kwargs):
        requires_backends(cls, {1})

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        requires_backends(cls, {1})
'''


__snake_case		:List[str]   	=  '''
def {0}(*args, **kwargs):
    requires_backends({0}, {1})
'''





def   __snake_case			(    _UpperCAmelCase       ):
				__a		 =       _re_backend.findall(_UpperCAmelCase       )
				if len(_UpperCAmelCase       ) == 0:
								return None

				return "_and_".join(_UpperCAmelCase       )





def   __snake_case			(    ):
				with open(os.path.join(_UpperCAmelCase     ,				'''__init__.py'''       )     ,				'''r'''     ,				encoding='''utf-8'''     ,				newline='''\n'''       ) as f:
								__a		 =       f.readlines()

				# Get to the point we do the actual imports for type checking
				__a		 =       0
				__a		 =       {}
				# Go through the end of the file
				while line_index < len(_UpperCAmelCase       ):
								# If the line contains is_backend_available, we grab all objects associated with the `else` block
								__a		 =       find_backend(lines[line_index]       )
								if backend is not None:
												while not lines[line_index].startswith('''else:'''       ):
																line_index += 1
												line_index += 1
												__a		 =       []
												# Until we unindent, add backend objects to the list
												while line_index < len(_UpperCAmelCase       ) and len(lines[line_index]       ) > 1:
																__a		 =       lines[line_index]
																__a		 =       _re_single_line_import.search(_UpperCAmelCase       )
																if single_line_import_search is not None:
																				objects.extend(single_line_import_search.groups()[0].split(''', '''       )       )
																elif line.startswith(''' ''' * 8       ):
																				objects.append(line[8:-2]       )
																line_index += 1

												if len(_UpperCAmelCase       ) > 0:
																__a		 =       objects
								else:
												line_index += 1

				return backend_specific_objects





def   __snake_case			(    _UpperCAmelCase     ,				_UpperCAmelCase       ):
				if name.isupper():
								return DUMMY_CONSTANT.format(_UpperCAmelCase       )
				elif name.islower():
								return DUMMY_FUNCTION.format(_UpperCAmelCase     ,				_UpperCAmelCase       )
				else:
								return DUMMY_CLASS.format(_UpperCAmelCase     ,				_UpperCAmelCase       )





def   __snake_case			(    _UpperCAmelCase=None       ):
				if backend_specific_objects is None:
								__a		 =       read_init()
				# For special correspondence backend to module name as used in the function requires_modulename
				__a		 =       {}

				for backend, objects in backend_specific_objects.items():
								__a		 =       '''[''' + ''', '''.join(f'"{b}"' for b in backend.split('''_and_'''       )       ) + ''']'''
								__a		 =       '''# This file is autogenerated by the command `make fix-copies`, do not edit.\n'''
								dummy_file += "from ..utils import DummyObject, requires_backends\n\n"
								dummy_file += "\n".join([create_dummy_object(_UpperCAmelCase     ,				_UpperCAmelCase       ) for o in objects]       )
								__a		 =       dummy_file

				return dummy_files





def   __snake_case			(    _UpperCAmelCase=False       ):
				__a		 =       create_dummy_files()
				# For special correspondence backend to shortcut as used in utils/dummy_xxx_objects.py
				__a		 =       {'''torch''': '''pt'''}

				# Locate actual dummy modules and read their content.
				__a		 =       os.path.join(_UpperCAmelCase     ,				'''utils'''       )
				__a		 =       {
				    backend: os.path.join(_UpperCAmelCase     ,				f'dummy_{short_names.get(_UpperCAmelCase     ,				_UpperCAmelCase       )}_objects.py'       )
				    for backend in dummy_files.keys()
				}

				__a		 =       {}
				for backend, file_path in dummy_file_paths.items():
								if os.path.isfile(_UpperCAmelCase       ):
												with open(_UpperCAmelCase     ,				'''r'''     ,				encoding='''utf-8'''     ,				newline='''\n'''       ) as f:
																__a		 =       f.read()
								else:
												__a		 =       ''''''

				for backend in dummy_files.keys():
								if dummy_files[backend] != actual_dummies[backend]:
												if overwrite:
																print(
																    f'Updating diffusers.utils.dummy_{short_names.get(_UpperCAmelCase     ,				_UpperCAmelCase       )}_objects.py as the main '
																    '''__init__ has new objects.'''       )
																with open(dummy_file_paths[backend]     ,				'''w'''     ,				encoding='''utf-8'''     ,				newline='''\n'''       ) as f:
																				f.write(dummy_files[backend]       )
												else:
																raise ValueError(
																    '''The main __init__ has objects that are not present in '''
																    f'diffusers.utils.dummy_{short_names.get(_UpperCAmelCase     ,				_UpperCAmelCase       )}_objects.py. Run `make fix-copies` '
																    '''to fix this.'''       )


if __name__ == "__main__":
	__snake_case		:int   	=  argparse.ArgumentParser()
	parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
	__snake_case		:Any   	=  parser.parse_args()

	check_dummies(args.fix_and_overwrite)










import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
__snake_case		:Tuple   	=  logging.getLogger(__name__)

if __name__ == "__main__":
 __snake_case		:Union[str, Any]   	=  argparse.ArgumentParser(
     description='''Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'''
 )
 parser.add_argument(
     '''--data_file''', type=str, default='''data/dump.bert-base-uncased.pickle''', help='''The binarized dataset.'''
 )
 parser.add_argument(
     '''--token_counts_dump''', type=str, default='''data/token_counts.bert-base-uncased.pickle''', help='''The dump file.'''
 )
 parser.add_argument('''--vocab_size''', default=3_0522, type=int)
 __snake_case		:List[str]   	=  parser.parse_args()

 logger.info(f'Loading data from {args.data_file}')
 with open(args.data_file, '''rb''') as fp:
  __snake_case		:Optional[Any]   	=  pickle.load(fp)

 logger.info('''Counting occurrences for MLM.''')
 __snake_case		:Dict   	=  Counter()
 for tk_ids in data:
  counter.update(tk_ids)
 __snake_case		:Optional[Any]   	=  [0] * args.vocab_size
 for k, v in counter.items():
  __snake_case		:Any   	=  v

 logger.info(f'Dump to {args.token_counts_dump}')
 with open(args.token_counts_dump, '''wb''') as handle:
  pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)





