






'''simple docstring'''





import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
snake_case    				=		"""src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
snake_case    				=		direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
snake_case    				=		re.compile(r"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
snake_case    				=		re.compile(r"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
snake_case    				=		re.compile(r"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
snake_case    				=		[
    ("""pretraining""", """MODEL_FOR_PRETRAINING_MAPPING_NAMES""", """AutoModelForPreTraining"""),
    ("""feature-extraction""", """MODEL_MAPPING_NAMES""", """AutoModel"""),
    ("""audio-classification""", """MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForAudioClassification"""),
    ("""text-generation""", """MODEL_FOR_CAUSAL_LM_MAPPING_NAMES""", """AutoModelForCausalLM"""),
    ("""automatic-speech-recognition""", """MODEL_FOR_CTC_MAPPING_NAMES""", """AutoModelForCTC"""),
    ("""image-classification""", """MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForImageClassification"""),
    ("""image-segmentation""", """MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES""", """AutoModelForImageSegmentation"""),
    ("""fill-mask""", """MODEL_FOR_MASKED_LM_MAPPING_NAMES""", """AutoModelForMaskedLM"""),
    ("""object-detection""", """MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES""", """AutoModelForObjectDetection"""),
    (
        """zero-shot-object-detection""",
        """MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES""",
        """AutoModelForZeroShotObjectDetection""",
    ),
    ("""question-answering""", """MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES""", """AutoModelForQuestionAnswering"""),
    ("""text2text-generation""", """MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES""", """AutoModelForSeq2SeqLM"""),
    ("""text-classification""", """MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForSequenceClassification"""),
    ("""automatic-speech-recognition""", """MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES""", """AutoModelForSpeechSeq2Seq"""),
    (
        """table-question-answering""",
        """MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES""",
        """AutoModelForTableQuestionAnswering""",
    ),
    ("""token-classification""", """MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForTokenClassification"""),
    ("""multiple-choice""", """MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES""", """AutoModelForMultipleChoice"""),
    (
        """next-sentence-prediction""",
        """MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES""",
        """AutoModelForNextSentencePrediction""",
    ),
    (
        """audio-frame-classification""",
        """MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES""",
        """AutoModelForAudioFrameClassification""",
    ),
    ("""audio-xvector""", """MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES""", """AutoModelForAudioXVector"""),
    (
        """document-question-answering""",
        """MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES""",
        """AutoModelForDocumentQuestionAnswering""",
    ),
    (
        """visual-question-answering""",
        """MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES""",
        """AutoModelForVisualQuestionAnswering""",
    ),
    ("""image-to-text""", """MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES""", """AutoModelForVision2Seq"""),
    (
        """zero-shot-image-classification""",
        """MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES""",
        """AutoModelForZeroShotImageClassification""",
    ),
    ("""depth-estimation""", """MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES""", """AutoModelForDepthEstimation"""),
    ("""video-classification""", """MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForVideoClassification"""),
    ("""mask-generation""", """MODEL_FOR_MASK_GENERATION_MAPPING_NAMES""", """AutoModelForMaskGeneration"""),
]


def   lowerCamelCase__			(     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      str							=    re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"						,     lowercase				)
  return [m.group(0				) for m in matches]


def   lowerCamelCase__			(     ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      List[Any]							=    transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
  SCREAMING_SNAKE_CASE       :      Any							=    {
      config.replace("Config"						,     ""				): model_type for model_type, config in config_maping_names.items()
  }

  # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
  SCREAMING_SNAKE_CASE       :      Dict							=    collections.defaultdict(lowercase				)
  SCREAMING_SNAKE_CASE       :      Optional[int]							=    collections.defaultdict(lowercase				)
  SCREAMING_SNAKE_CASE       :      str							=    collections.defaultdict(lowercase				)

  # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
  for attr_name in dir(lowercase				):
    SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    None
    if _re_tf_models.match(lowercase				) is not None:
      SCREAMING_SNAKE_CASE       :      Optional[Any]							=    tf_models
      SCREAMING_SNAKE_CASE       :      Optional[int]							=    _re_tf_models.match(lowercase				).groups()[0]
    elif _re_flax_models.match(lowercase				) is not None:
      SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    flax_models
      SCREAMING_SNAKE_CASE       :      List[str]							=    _re_flax_models.match(lowercase				).groups()[0]
    elif _re_pt_models.match(lowercase				) is not None:
      SCREAMING_SNAKE_CASE       :      Dict							=    pt_models
      SCREAMING_SNAKE_CASE       :      Dict							=    _re_pt_models.match(lowercase				).groups()[0]

    if lookup_dict is not None:
      while len(lowercase				) > 0:
        if attr_name in model_prefix_to_model_type:
          SCREAMING_SNAKE_CASE       :      Optional[int]							=    True
          break
        # Try again after removing the last word in the name
        SCREAMING_SNAKE_CASE       :      str							=    "".join(camel_case_split(lowercase				)[:-1]				)

  SCREAMING_SNAKE_CASE       :      Tuple							=    set(list(pt_models.keys()				) + list(tf_models.keys()				) + list(flax_models.keys()				)				)
  SCREAMING_SNAKE_CASE       :      List[str]							=    list(lowercase				)
  all_models.sort()

  SCREAMING_SNAKE_CASE       :      str							=    {"model_type": all_models}
  SCREAMING_SNAKE_CASE       :      int							=    [pt_models[t] for t in all_models]
  SCREAMING_SNAKE_CASE       :      Tuple							=    [tf_models[t] for t in all_models]
  SCREAMING_SNAKE_CASE       :      Any							=    [flax_models[t] for t in all_models]

  # Now let's use the auto-mapping names to make sure
  SCREAMING_SNAKE_CASE       :      str							=    {}
  for t in all_models:
    if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
      SCREAMING_SNAKE_CASE       :      Dict							=    "AutoProcessor"
    elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
      SCREAMING_SNAKE_CASE       :      Optional[int]							=    "AutoTokenizer"
    elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
      SCREAMING_SNAKE_CASE       :      Optional[Any]							=    "AutoFeatureExtractor"
    else:
      # Default to AutoTokenizer if a model has nothing, for backward compatibility.
      SCREAMING_SNAKE_CASE       :      Optional[int]							=    "AutoTokenizer"

  SCREAMING_SNAKE_CASE       :      Dict							=    [processors[t] for t in all_models]

  return pd.DataFrame(lowercase				)


def   lowerCamelCase__			(     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Optional[int]							=    [
      transformers_module.models.auto.modeling_auto,
      transformers_module.models.auto.modeling_tf_auto,
      transformers_module.models.auto.modeling_flax_auto,
  ]
  for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
    SCREAMING_SNAKE_CASE       :      List[str]							=    [model_mapping, F'''TF_{model_mapping}''', F'''FLAX_{model_mapping}''']
    SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    [auto_class, F'''TF_{auto_class}''', F'''Flax_{auto_class}''']
    # Loop through all three frameworks
    for module, cls, mapping in zip(lowercase						,     lowercase						,     lowercase				):
      # The type of pipeline may not exist in this framework
      if not hasattr(lowercase						,     lowercase				):
        continue
      # First extract all model_names
      SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    []
      for name in getattr(lowercase						,     lowercase				).values():
        if isinstance(lowercase						,     lowercase				):
          model_names.append(lowercase				)
        else:
          model_names.extend(list(lowercase				)				)

            # Add pipeline tag and auto model class for those models
      table.update({model_name: (pipeline_tag, cls) for model_name in model_names}				)

  return table


def   lowerCamelCase__			(     lowercase						,     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      List[Any]							=    get_frameworks_table()
  SCREAMING_SNAKE_CASE       :      Any							=    Dataset.from_pandas(lowercase				)

  SCREAMING_SNAKE_CASE       :      Dict							=    hf_hub_download(
      "huggingface/transformers-metadata"						,     "pipeline_tags.json"						,     repo_type="dataset"						,     token=lowercase				)
  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    Dataset.from_json(lowercase				)
  SCREAMING_SNAKE_CASE       :      Optional[int]							=    {
      tags_dataset[i]["model_class"]: (tags_dataset[i]["pipeline_tag"], tags_dataset[i]["auto_class"])
      for i in range(len(lowercase				)				)
  }
  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    update_pipeline_and_auto_class_table(lowercase				)

  # Sort the model classes to avoid some nondeterministic updates to create false update commits.
  SCREAMING_SNAKE_CASE       :      Any							=    sorted(table.keys()				)
  SCREAMING_SNAKE_CASE       :      Dict							=    pd.DataFrame(
      {
          "model_class": model_classes,
          "pipeline_tag": [table[m][0] for m in model_classes],
          "auto_class": [table[m][1] for m in model_classes],
      }				)
  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    Dataset.from_pandas(lowercase				)

  with tempfile.TemporaryDirectory() as tmp_dir:
    frameworks_dataset.to_json(os.path.join(lowercase						,     "frameworks.json"				)				)
    tags_dataset.to_json(os.path.join(lowercase						,     "pipeline_tags.json"				)				)

    if commit_sha is not None:
      SCREAMING_SNAKE_CASE       :      Any							=    (
          F'''Update with commit {commit_sha}\n\nSee: '''
          F'''https://github.com/huggingface/transformers/commit/{commit_sha}'''
      )
    else:
      SCREAMING_SNAKE_CASE       :      str							=    "Update"

    upload_folder(
        repo_id="huggingface/transformers-metadata"						,     folder_path=lowercase						,     repo_type="dataset"						,     token=lowercase						,     commit_message=lowercase						,     )


def   lowerCamelCase__			(     ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Dict							=    {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    transformers_module.pipelines.SUPPORTED_TASKS
  SCREAMING_SNAKE_CASE       :      List[Any]							=    []
  for key in pipeline_tasks:
    if key not in in_table:
      SCREAMING_SNAKE_CASE       :      str							=    pipeline_tasks[key]["pt"]
      if isinstance(lowercase						,     (list, tuple)				):
        SCREAMING_SNAKE_CASE       :      Optional[Any]							=    model[0]
      SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    model.__name__
      if model not in in_table.values():
        missing.append(lowercase				)

  if len(lowercase				) > 0:
    SCREAMING_SNAKE_CASE       :      str							=    ", ".join(lowercase				)
    raise ValueError(
        "The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside "
        F'''`utils/update_metadata.py`: {msg}. Please add them!'''				)


if __name__ == "__main__":
   snake_case    				=		argparse.ArgumentParser()
   parser.add_argument("""--token""", type=str, help="""The token to use to push to the transformers-metadata dataset.""")
   parser.add_argument("""--commit_sha""", type=str, help="""The sha of the commit going with this update.""")
   parser.add_argument("""--check-only""", action="""store_true""", help="""Activate to just check all pipelines are present.""")
   snake_case    				=		parser.parse_args()

   if args.check_only:
      check_pipeline_tags()
   else:
      update_metadata(args.token, args.commit_sha)














from transformers import HfArgumentParser, TensorFlowBenchmark, TensorFlowBenchmarkArguments


def   lowerCamelCase__			(     ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      List[Any]							=    HfArgumentParser(lowercase				)
  SCREAMING_SNAKE_CASE       :      Any							=    parser.parse_args_into_dataclasses()[0]
  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    TensorFlowBenchmark(args=lowercase				)
  try:
    SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    parser.parse_args_into_dataclasses()[0]
  except ValueError as e:
    SCREAMING_SNAKE_CASE       :      int							=    "Arg --no_{0} is no longer used, please use --no-{0} instead."
    SCREAMING_SNAKE_CASE       :      Optional[int]							=    " ".join(str(lowercase				).split(" "				)[:-1]				)
    SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    ""
    SCREAMING_SNAKE_CASE       :      Any							=    eval(str(lowercase				).split(" "				)[-1]				)
    SCREAMING_SNAKE_CASE       :      List[str]							=    []
    for arg in depreciated_args:
      # arg[2:] removes '--'
      if arg[2:] in TensorFlowBenchmark.deprecated_args:
        # arg[5:] removes '--no_'
        full_error_msg += arg_error_msg.format(arg[5:]				)
      else:
        wrong_args.append(lowercase				)
    if len(lowercase				) > 0:
      SCREAMING_SNAKE_CASE       :      Optional[int]							=    full_error_msg + begin_error_msg + str(lowercase				)
    raise ValueError(lowercase				)
  benchmark.run()


if __name__ == "__main__":
   main()






