




import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
__lowerCamelCase   :						str				=			'''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
__lowerCamelCase   :						Optional[int]				=			direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
__lowerCamelCase   :						Union[str, Any]				=			re.compile(R'''TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
__lowerCamelCase   :						Tuple				=			re.compile(R'''Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
__lowerCamelCase   :						Tuple				=			re.compile(R'''(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
__lowerCamelCase   :						Dict				=			[
    ('''pretraining''', '''MODEL_FOR_PRETRAINING_MAPPING_NAMES''', '''AutoModelForPreTraining'''),
    ('''feature-extraction''', '''MODEL_MAPPING_NAMES''', '''AutoModel'''),
    ('''audio-classification''', '''MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForAudioClassification'''),
    ('''text-generation''', '''MODEL_FOR_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForCausalLM'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_CTC_MAPPING_NAMES''', '''AutoModelForCTC'''),
    ('''image-classification''', '''MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForImageClassification'''),
    ('''image-segmentation''', '''MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES''', '''AutoModelForImageSegmentation'''),
    ('''fill-mask''', '''MODEL_FOR_MASKED_LM_MAPPING_NAMES''', '''AutoModelForMaskedLM'''),
    ('''object-detection''', '''MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES''', '''AutoModelForObjectDetection'''),
    (
        '''zero-shot-object-detection''',
        '''MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES''',
        '''AutoModelForZeroShotObjectDetection''',
    ),
    ('''question-answering''', '''MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES''', '''AutoModelForQuestionAnswering'''),
    ('''text2text-generation''', '''MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForSeq2SeqLM'''),
    ('''text-classification''', '''MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForSequenceClassification'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES''', '''AutoModelForSpeechSeq2Seq'''),
    (
        '''table-question-answering''',
        '''MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForTableQuestionAnswering''',
    ),
    ('''token-classification''', '''MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForTokenClassification'''),
    ('''multiple-choice''', '''MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES''', '''AutoModelForMultipleChoice'''),
    (
        '''next-sentence-prediction''',
        '''MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES''',
        '''AutoModelForNextSentencePrediction''',
    ),
    (
        '''audio-frame-classification''',
        '''MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForAudioFrameClassification''',
    ),
    ('''audio-xvector''', '''MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES''', '''AutoModelForAudioXVector'''),
    (
        '''document-question-answering''',
        '''MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForDocumentQuestionAnswering''',
    ),
    (
        '''visual-question-answering''',
        '''MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForVisualQuestionAnswering''',
    ),
    ('''image-to-text''', '''MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES''', '''AutoModelForVision2Seq'''),
    (
        '''zero-shot-image-classification''',
        '''MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForZeroShotImageClassification''',
    ),
    ('''depth-estimation''', '''MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES''', '''AutoModelForDepthEstimation'''),
    ('''video-classification''', '''MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForVideoClassification'''),
    ('''mask-generation''', '''MODEL_FOR_MASK_GENERATION_MAPPING_NAMES''', '''AutoModelForMaskGeneration'''),
]







def 		lowercase__   (		__A:	Optional[Any]				):





     '''simple docstring'''






     __magic_name__							:	List[Any]    							=							re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'''							,__A				)
     return [m.group(0				) for m in matches]







def 		lowercase__   (		):





     '''simple docstring'''






     __magic_name__							:	List[str]    							=							transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
     __magic_name__							:	str    							=							{
         config.replace('''Config'''							,''''''				): model_type for model_type, config in config_maping_names.items()
     }

     # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
     __magic_name__							:	int    							=							collections.defaultdict(__A				)
     __magic_name__							:	Dict    							=							collections.defaultdict(__A				)
     __magic_name__							:	Optional[int]    							=							collections.defaultdict(__A				)

     # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
     for attr_name in dir(__A				):
          __magic_name__							:	Optional[Any]    							=							None
          if _re_tf_models.match(__A				) is not None:
               __magic_name__							:	Optional[int]    							=							tf_models
               __magic_name__							:	str    							=							_re_tf_models.match(__A				).groups()[0]
          elif _re_flax_models.match(__A				) is not None:
               __magic_name__							:	Any    							=							flax_models
               __magic_name__							:	Dict    							=							_re_flax_models.match(__A				).groups()[0]
          elif _re_pt_models.match(__A				) is not None:
               __magic_name__							:	List[str]    							=							pt_models
               __magic_name__							:	Union[str, Any]    							=							_re_pt_models.match(__A				).groups()[0]

          if lookup_dict is not None:
               while len(__A				) > 0:
                    if attr_name in model_prefix_to_model_type:
                         __magic_name__							:	str    							=							True
                         break
                    # Try again after removing the last word in the name
                    __magic_name__							:	Tuple    							=							''''''.join(camel_case_split(__A				)[:-1]				)

     __magic_name__							:	Optional[Any]    							=							set(list(pt_models.keys()				) + list(tf_models.keys()				) + list(flax_models.keys()				)				)
     __magic_name__							:	str    							=							list(__A				)
     all_models.sort()

     __magic_name__							:	Any    							=							{'''model_type''': all_models}
     __magic_name__							:	List[str]    							=							[pt_models[t] for t in all_models]
     __magic_name__							:	Any    							=							[tf_models[t] for t in all_models]
     __magic_name__							:	List[Any]    							=							[flax_models[t] for t in all_models]

     # Now let's use the auto-mapping names to make sure
     __magic_name__							:	Optional[Any]    							=							{}
     for t in all_models:
          if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
               __magic_name__							:	List[str]    							=							'''AutoProcessor'''
          elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
               __magic_name__							:	List[Any]    							=							'''AutoTokenizer'''
          elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
               __magic_name__							:	Dict    							=							'''AutoFeatureExtractor'''
          else:
               # Default to AutoTokenizer if a model has nothing, for backward compatibility.
               __magic_name__							:	Dict    							=							'''AutoTokenizer'''

     __magic_name__							:	Optional[int]    							=							[processors[t] for t in all_models]

     return pd.DataFrame(__A				)







def 		lowercase__   (		__A:	Tuple				):





     '''simple docstring'''






     __magic_name__							:	List[Any]    							=							[
         transformers_module.models.auto.modeling_auto,
         transformers_module.models.auto.modeling_tf_auto,
         transformers_module.models.auto.modeling_flax_auto,
     ]
     for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
          __magic_name__							:	int    							=							[model_mapping, F'''TF_{model_mapping}''', F'''FLAX_{model_mapping}''']
          __magic_name__							:	Dict    							=							[auto_class, F'''TF_{auto_class}''', F'''Flax_{auto_class}''']
          # Loop through all three frameworks
          for module, cls, mapping in zip(__A							,__A							,__A				):
               # The type of pipeline may not exist in this framework
               if not hasattr(__A							,__A				):
                    continue
               # First extract all model_names
               __magic_name__							:	int    							=							[]
               for name in getattr(__A							,__A				).values():
                    if isinstance(__A							,__A				):
                         model_names.append(__A				)
                    else:
                         model_names.extend(list(__A				)				)

            # Add pipeline tag and auto model class for those models
               table.update({model_name: (pipeline_tag, cls) for model_name in model_names}				)

     return table







def 		lowercase__   (		__A:	List[str]							,__A:	Optional[Any]				):





     '''simple docstring'''






     __magic_name__							:	Optional[Any]    							=							get_frameworks_table()
     __magic_name__							:	Union[str, Any]    							=							Dataset.from_pandas(__A				)

     __magic_name__							:	int    							=							hf_hub_download(
         '''huggingface/transformers-metadata'''							,'''pipeline_tags.json'''							,repo_type='''dataset'''							,token=__A				)
     __magic_name__							:	List[Any]    							=							Dataset.from_json(__A				)
     __magic_name__							:	List[str]    							=							{
         tags_dataset[i]['''model_class''']: (tags_dataset[i]['''pipeline_tag'''], tags_dataset[i]['''auto_class'''])
         for i in range(len(__A				)				)
     }
     __magic_name__							:	Optional[int]    							=							update_pipeline_and_auto_class_table(__A				)

     # Sort the model classes to avoid some nondeterministic updates to create false update commits.
     __magic_name__							:	int    							=							sorted(table.keys()				)
     __magic_name__							:	Any    							=							pd.DataFrame(
         {
             '''model_class''': model_classes,
             '''pipeline_tag''': [table[m][0] for m in model_classes],
             '''auto_class''': [table[m][1] for m in model_classes],
         }				)
     __magic_name__							:	List[str]    							=							Dataset.from_pandas(__A				)

     with tempfile.TemporaryDirectory() as tmp_dir:
          frameworks_dataset.to_json(os.path.join(__A							,'''frameworks.json'''				)				)
          tags_dataset.to_json(os.path.join(__A							,'''pipeline_tags.json'''				)				)

          if commit_sha is not None:
               __magic_name__							:	Optional[Any]    							=							(
                   F'''Update with commit {commit_sha}\n\nSee: '''
                   F'''https://github.com/huggingface/transformers/commit/{commit_sha}'''
               )
          else:
               __magic_name__							:	List[Any]    							=							'''Update'''

          upload_folder(
              repo_id='''huggingface/transformers-metadata'''							,folder_path=__A							,repo_type='''dataset'''							,token=__A							,commit_message=__A							,)







def 		lowercase__   (		):





     '''simple docstring'''






     __magic_name__							:	Dict    							=							{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
     __magic_name__							:	str    							=							transformers_module.pipelines.SUPPORTED_TASKS
     __magic_name__							:	Dict    							=							[]
     for key in pipeline_tasks:
          if key not in in_table:
               __magic_name__							:	List[str]    							=							pipeline_tasks[key]['''pt''']
               if isinstance(__A							,(list, tuple)				):
                    __magic_name__							:	Optional[int]    							=							model[0]
               __magic_name__							:	int    							=							model.__name__
               if model not in in_table.values():
                    missing.append(__A				)

     if len(__A				) > 0:
          __magic_name__							:	Union[str, Any]    							=							''', '''.join(__A				)
          raise ValueError(
              '''The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '''
              F'''`utils/update_metadata.py`: {msg}. Please add them!'''				)


if __name__ == "__main__":
 __lowerCamelCase   :						int				=			argparse.ArgumentParser()
 parser.add_argument('''--token''', type=str, help='''The token to use to push to the transformers-metadata dataset.''')
 parser.add_argument('''--commit_sha''', type=str, help='''The sha of the commit going with this update.''')
 parser.add_argument('''--check-only''', action='''store_true''', help='''Activate to just check all pipelines are present.''')
 __lowerCamelCase   :						List[str]				=			parser.parse_args()

 if args.check_only:
  check_pipeline_tags()
 else:
  update_metadata(args.token, args.commit_sha)









import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command


class    lowerCamelCase    (		unittest.TestCase						):






			'''simple docstring'''





			UpperCamelCase__								=inspect.getfile(accelerate.test_utils						)
			UpperCamelCase__								=os.path.sep.join(mod_file.split(os.path.sep						)[:-1] + ['''scripts''', '''test_cli.py''']						)

			UpperCamelCase__								=['''accelerate''', '''launch''']
			UpperCamelCase__								=Path.home() / '''.cache/huggingface/accelerate'''
			UpperCamelCase__								='''default_config.yaml'''
			UpperCamelCase__								=config_folder / config_file
			UpperCamelCase__								=config_folder / '''_default_config.yaml'''

			UpperCamelCase__								=Path('''tests/test_configs'''						)

			@classmethod
			def 							UpperCAmelCase__       ( cls			:   Optional[int]  )						->   int:
								if cls.config_path.is_file():
													cls.config_path.rename(cls.changed_path  )

			@classmethod
			def 							UpperCAmelCase__       ( cls			:   Dict  )						->   List[str]:
								if cls.changed_path.is_file():
													cls.changed_path.rename(cls.config_path  )

			def 							UpperCAmelCase__       ( self			:   str  )						->   Optional[Any]:
								__magic_name__							:	Dict    							=							self.base_cmd
								if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
													cmd += ["--multi_gpu"]
								execute_subprocess_async(cmd + [self.test_file_path]       ,						env=os.environ.copy()  )

			def 							UpperCAmelCase__       ( self			:   int  )						->   Optional[int]:
								for config in sorted(self.test_config_path.glob('''**/*.yaml'''  )  ):
													with self.subTest(config_file=lowerCamelCase_  ):
																		execute_subprocess_async(
																		    self.base_cmd + ['''--config_file''', str(lowerCamelCase_  ), self.test_file_path]       ,						env=os.environ.copy()  )


			def 							UpperCAmelCase__       ( self			:   int  )						->   Union[str, Any]:
								execute_subprocess_async(['''accelerate''', '''test''']       ,						env=os.environ.copy()  )






class    lowerCamelCase    (		unittest.TestCase						):






			'''simple docstring'''





			UpperCamelCase__								='''test-tpu'''
			UpperCamelCase__								='''us-central1-a'''
			UpperCamelCase__								='''ls'''
			UpperCamelCase__								=['''accelerate''', '''tpu-config''']
			UpperCamelCase__								='''cd /usr/share'''
			UpperCamelCase__								='''tests/test_samples/test_command_file.sh'''
			UpperCamelCase__								='''Running gcloud compute tpus tpu-vm ssh'''

			def 							UpperCAmelCase__       ( self			:   Optional[Any]  )						->   Optional[Any]:
								__magic_name__							:	str    							=							run_command(
								    self.cmd
								    + ['''--command''', self.command, '''--tpu_zone''', self.tpu_zone, '''--tpu_name''', self.tpu_name, '''--debug''']       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   Union[str, Any]  )						->   int:
								__magic_name__							:	int    							=							run_command(
								    self.cmd
								    + [
								        '''--config_file''',
								        '''tests/test_configs/0_12_0.yaml''',
								        '''--command''',
								        self.command,
								        '''--tpu_zone''',
								        self.tpu_zone,
								        '''--tpu_name''',
								        self.tpu_name,
								        '''--debug''',
								    ]       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   List[Any]  )						->   Tuple:
								__magic_name__							:	Any    							=							run_command(
								    self.cmd + ['''--config_file''', '''tests/test_configs/latest.yaml''', '''--debug''']       ,						return_stdout=lowerCamelCase_  )
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   Optional[int]  )						->   Tuple:
								__magic_name__							:	Tuple    							=							run_command(
								    self.cmd + ['''--config_file''', '''tests/test_configs/latest.yaml''', '''--command''', self.command, '''--debug''']       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   List[Any]  )						->   Dict:
								__magic_name__							:	str    							=							run_command(
								    self.cmd
								    + [
								        '''--config_file''',
								        '''tests/test_configs/latest.yaml''',
								        '''--command''',
								        self.command,
								        '''--command''',
								        '''echo "Hello World"''',
								        '''--debug''',
								    ]       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo "Hello World" --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   Optional[Any]  )						->   Optional[int]:
								__magic_name__							:	Union[str, Any]    							=							run_command(
								    self.cmd
								    + ['''--config_file''', '''tests/test_configs/latest.yaml''', '''--command_file''', self.command_file, '''--debug''']       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   int  )						->   List[Any]:
								__magic_name__							:	List[Any]    							=							run_command(
								    self.cmd
								    + [
								        '''--config_file''',
								        '''tests/test_configs/0_12_0.yaml''',
								        '''--command_file''',
								        self.command_file,
								        '''--tpu_zone''',
								        self.tpu_zone,
								        '''--tpu_name''',
								        self.tpu_name,
								        '''--debug''',
								    ]       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   str  )						->   Dict:
								__magic_name__							:	Tuple    							=							run_command(
								    self.cmd + ['''--config_file''', '''tests/test_configs/latest.yaml''', '''--install_accelerate''', '''--debug''']       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo "hello world"; echo "this is a second command" --worker all'''       ,						lowerCamelCase_       ,						)


			def 							UpperCAmelCase__       ( self			:   Any  )						->   Tuple:
								__magic_name__							:	Tuple    							=							run_command(
								    self.cmd
								    + [
								        '''--config_file''',
								        '''tests/test_configs/latest.yaml''',
								        '''--install_accelerate''',
								        '''--accelerate_version''',
								        '''12.0.0''',
								        '''--debug''',
								    ]       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo "hello world"; echo "this is a second command" --worker all'''       ,						lowerCamelCase_       ,						)



