
import argparse
import os
import re


lowerCamelCase__    :		List[Any]          =   """src/transformers"""

# Pattern that looks at the indentation in a line.
lowerCamelCase__    :		Union[str, Any]          =   re.compile(R"""^(\s*)\S""")
# Pattern that matches `"key":" and puts `key` in group 0.
lowerCamelCase__    :		Any          =   re.compile(R"""^\s*\"([^\"]+)\":""")
# Pattern that matches `_import_structure["key"]` and puts `key` in group 0.
lowerCamelCase__    :		int          =   re.compile(R"""^\s*_import_structure\[\"([^\"]+)\"\]""")
# Pattern that matches `"key",` and puts `key` in group 0.
lowerCamelCase__    :		List[Any]          =   re.compile(R"""^\s*\"([^\"]+)\",\s*$""")
# Pattern that matches any `[stuff]` and puts `stuff` in group 0.
lowerCamelCase__    :		str          =   re.compile(R"""\[([^\]]+)\]""")

def 		UpperCamelCase	(      lowercase_   )		->	Union[str, Any]:


  '''simple docstring'''






  lowercase__  :   Optional[Any]    		=      _re_indent.search(lowercase_   )
  return "" if search is None else search.groups()[0]

def 		UpperCamelCase	(      lowercase_ , lowercase_="" , lowercase_=None , lowercase_=None   )		->	Dict:


  '''simple docstring'''






  lowercase__  :   List[str]    		=      0
  lowercase__  :   List[Any]    		=      code.split("""\n"""   )
  if start_prompt is not None:
    while not lines[index].startswith(lowercase_   ):
      index += 1
    lowercase__  :   List[str]    		=      ["""\n""".join(lines[:index]   )]
  else:
    lowercase__  :   Optional[Any]    		=      []

  # We split into blocks until we get to the `end_prompt` (or the end of the block).
  lowercase__  :   List[Any]    		=      [lines[index]]
  index += 1
  while index < len(lowercase_   ) and (end_prompt is None or not lines[index].startswith(lowercase_   )):
    if len(lines[index]   ) > 0 and get_indent(lines[index]   ) == indent_level:
      if len(lowercase_   ) > 0 and get_indent(current_block[-1]   ).startswith(indent_level + """ """   ):
        current_block.append(lines[index]   )
        blocks.append("""\n""".join(lowercase_   )   )
        if index < len(lowercase_   ) - 1:
          lowercase__  :   str    		=      [lines[index + 1]]
          index += 1
        else:
          lowercase__  :   Union[str, Any]    		=      []
      else:
        blocks.append("""\n""".join(lowercase_   )   )
        lowercase__  :   Tuple    		=      [lines[index]]
    else:
      current_block.append(lines[index]   )
    index += 1

  # Adds current block if it's nonempty.
  if len(lowercase_   ) > 0:
    blocks.append("""\n""".join(lowercase_   )   )

  # Add final block after end_prompt if provided.
  if end_prompt is not None and index < len(lowercase_   ):
    blocks.append("""\n""".join(lines[index:]   )   )

  return blocks

def 		UpperCamelCase	(      lowercase_   )		->	List[Any]:


  '''simple docstring'''






  def _inner(lowercase_   ):
    return key(lowercase_   ).lower().replace("""_""" , """"""   )

  return _inner

def 		UpperCamelCase	(      lowercase_ , lowercase_=None   )		->	Optional[Any]:


  '''simple docstring'''






  def noop(lowercase_   ):
    return x

  if key is None:
    lowercase__  :   Dict    		=      noop
  # Constants are all uppercase, they go first.
  lowercase__  :   Dict    		=      [obj for obj in objects if key(lowercase_   ).isupper()]
  # Classes are not all uppercase but start with a capital, they go second.
  lowercase__  :   Any    		=      [obj for obj in objects if key(lowercase_   )[0].isupper() and not key(lowercase_   ).isupper()]
  # Functions begin with a lowercase, they go last.
  lowercase__  :   Optional[int]    		=      [obj for obj in objects if not key(lowercase_   )[0].isupper()]

  lowercase__  :   Tuple    		=      ignore_underscore(lowercase_   )
  return sorted(lowercase_ , key=lowercase_   ) + sorted(lowercase_ , key=lowercase_   ) + sorted(lowercase_ , key=lowercase_   )

def 		UpperCamelCase	(      lowercase_   )		->	Tuple:


  '''simple docstring'''






  def _replace(lowercase_   ):
    lowercase__  :   int    		=      match.groups()[0]
    if "," not in imports:
      return F'[{imports}]'
    lowercase__  :   Union[str, Any]    		=      [part.strip().replace("""\"""" , """"""   ) for part in imports.split(""","""   )]
    # We will have a final empty element if the line finished with a comma.
    if len(keys[-1]   ) == 0:
      lowercase__  :   int    		=      keys[:-1]
    return "[" + ", ".join([F'"{k}"' for k in sort_objects(lowercase_   )]   ) + "]"

  lowercase__  :   Any    		=      import_statement.split("""\n"""   )
  if len(lowercase_   ) > 3:
    # Here we have to sort internal imports that are on several lines (one per name):
    # key: [
    #     "object1",
    #     "object2",
    #     ...
    # ]

    # We may have to ignore one or two lines on each side.
    lowercase__  :   Tuple    		=      2 if lines[1].strip() == """[""" else 1
    lowercase__  :   Optional[int]    		=      [(i, _re_strip_line.search(lowercase_   ).groups()[0]) for i, line in enumerate(lines[idx:-idx]   )]
    lowercase__  :   Optional[Any]    		=      sort_objects(lowercase_ , key=lambda lowercase_   : x[1]   )
    lowercase__  :   Union[str, Any]    		=      [lines[x[0] + idx] for x in sorted_indices]
    return "\n".join(lines[:idx] + sorted_lines + lines[-idx:]   )
  elif len(lowercase_   ) == 3:
    # Here we have to sort internal imports that are on one separate line:
    # key: [
    #     "object1", "object2", ...
    # ]
    if _re_bracket_content.search(lines[1]   ) is not None:
      lowercase__  :   List[str]    		=      _re_bracket_content.sub(_replace , lines[1]   )
    else:
      lowercase__  :   Optional[int]    		=      [part.strip().replace("""\"""" , """"""   ) for part in lines[1].split(""","""   )]
      # We will have a final empty element if the line finished with a comma.
      if len(keys[-1]   ) == 0:
        lowercase__  :   Optional[int]    		=      keys[:-1]
      lowercase__  :   int    		=      get_indent(lines[1]   ) + """, """.join([F'"{k}"' for k in sort_objects(lowercase_   )]   )
    return "\n".join(lowercase_   )
  else:
    # Finally we have to deal with imports fitting on one line
    lowercase__  :   Any    		=      _re_bracket_content.sub(_replace , lowercase_   )
    return import_statement

def 		UpperCamelCase	(      lowercase_ , lowercase_=True   )		->	Optional[int]:


  '''simple docstring'''






  with open(lowercase_ , encoding="""utf-8"""   ) as f:
    lowercase__  :   str    		=      f.read()

  if "_import_structure" not in code:
    return

  # Blocks of indent level 0
  lowercase__  :   Any    		=      split_code_in_indented_blocks(
      lowercase_ , start_prompt="""_import_structure = {""" , end_prompt="""if TYPE_CHECKING:"""   )

  # We ignore block 0 (everything untils start_prompt) and the last block (everything after end_prompt).
  for block_idx in range(1 , len(lowercase_   ) - 1   ):
    # Check if the block contains some `_import_structure`s thingy to sort.
    lowercase__  :   List[str]    		=      main_blocks[block_idx]
    lowercase__  :   str    		=      block.split("""\n"""   )

    # Get to the start of the imports.
    lowercase__  :   Optional[Any]    		=      0
    while line_idx < len(lowercase_   ) and "_import_structure" not in block_lines[line_idx]:
      # Skip dummy import blocks
      if "import dummy" in block_lines[line_idx]:
        lowercase__  :   int    		=      len(lowercase_   )
      else:
        line_idx += 1
    if line_idx >= len(lowercase_   ):
      continue

    # Ignore beginning and last line: they don't contain anything.
    lowercase__  :   List[str]    		=      """\n""".join(block_lines[line_idx:-1]   )
    lowercase__  :   Dict    		=      get_indent(block_lines[1]   )
    # Slit the internal block into blocks of indent level 1.
    lowercase__  :   List[str]    		=      split_code_in_indented_blocks(lowercase_ , indent_level=lowercase_   )
    # We have two categories of import key: list or _import_structure[key].append/extend
    lowercase__  :   Optional[int]    		=      _re_direct_key if """_import_structure = {""" in block_lines[0] else _re_indirect_key
    # Grab the keys, but there is a trap: some lines are empty or just comments.
    lowercase__  :   Optional[Any]    		=      [(pattern.search(lowercase_   ).groups()[0] if pattern.search(lowercase_   ) is not None else None) for b in internal_blocks]
    # We only sort the lines with a key.
    lowercase__  :   List[Any]    		=      [(i, key) for i, key in enumerate(lowercase_   ) if key is not None]
    lowercase__  :   Optional[int]    		=      [x[0] for x in sorted(lowercase_ , key=lambda lowercase_   : x[1]   )]

    # We reorder the blocks by leaving empty lines/comments as they were and reorder the rest.
    lowercase__  :   Any    		=      0
    lowercase__  :   Optional[Any]    		=      []
    for i in range(len(lowercase_   )   ):
      if keys[i] is None:
        reorderded_blocks.append(internal_blocks[i]   )
      else:
        lowercase__  :   Optional[Any]    		=      sort_objects_in_import(internal_blocks[sorted_indices[count]]   )
        reorderded_blocks.append(lowercase_   )
        count += 1

        # And we put our main block back together with its first and last line.
    lowercase__  :   Union[str, Any]    		=      """\n""".join(block_lines[:line_idx] + reorderded_blocks + [block_lines[-1]]   )

  if code != "\n".join(lowercase_   ):
    if check_only:
      return True
    else:
      print(F'Overwriting {file}.'   )
      with open(lowercase_ , """w""" , encoding="""utf-8"""   ) as f:
        f.write("""\n""".join(lowercase_   )   )

def 		UpperCamelCase	(      lowercase_=True   )		->	Optional[int]:


  '''simple docstring'''






  lowercase__  :   int    		=      []
  for root, _, files in os.walk(lowercase_   ):
    if "__init__.py" in files:
      lowercase__  :   Dict    		=      sort_imports(os.path.join(lowercase_ , """__init__.py"""   ) , check_only=lowercase_   )
      if result:
        lowercase__  :   List[str]    		=      [os.path.join(lowercase_ , """__init__.py"""   )]
  if len(lowercase_   ) > 0:
    raise ValueError(F'Would overwrite {len(lowercase_   )} files, run `make style`.'   )


if __name__ == "__main__":
     lowerCamelCase__    :		int          =   argparse.ArgumentParser()
     parser.add_argument("""--check_only""", action="""store_true""", help="""Whether to only check or fix style.""")
     lowerCamelCase__    :		List[Any]          =   parser.parse_args()

     sort_imports_in_all_inits(check_only=args.check_only)








from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowerCamelCase__    :		Dict          =   """\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

lowerCamelCase__    :		Tuple          =   """\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

lowerCamelCase__    :		Any          =   """
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Optional[Any]:


  '''simple docstring'''






  return float((preds == labels).mean()   )

def 		UpperCamelCase	(      lowercase_ , lowercase_ , lowercase_="binary"   )		->	int:


  '''simple docstring'''






  lowercase__  :   List[str]    		=      simple_accuracy(lowercase_ , lowercase_   )
  lowercase__  :   Optional[Any]    		=      float(fa_score(y_true=lowercase_ , y_pred=lowercase_ , average=lowercase_   )   )
  return {
      "accuracy": acc,
      "f1": fa,
  }

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Dict:


  '''simple docstring'''






  lowercase__  :   List[Any]    		=      {}
  for id_pred, label in zip(lowercase_ , lowercase_   ):
    lowercase__  :   Optional[Any]    		=      F'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
    lowercase__  :   Optional[Any]    		=      id_pred["""prediction"""]
    if question_id in question_map:
      question_map[question_id].append((pred, label)   )
    else:
      lowercase__  :   Optional[int]    		=      [(pred, label)]
  lowercase__		,    lowercase__  :   List[str]    		=      [], []
  for question, preds_labels in question_map.items():
    lowercase__		,    lowercase__  :   List[str]    		=      zip(*lowercase_   )
    lowercase__  :   Optional[Any]    		=      fa_score(y_true=lowercase_ , y_pred=lowercase_ , average="""macro"""   )
    fas.append(lowercase_   )
    lowercase__  :   List[Any]    		=      int(sum(pred == label for pred, label in preds_labels   ) == len(lowercase_   )   )
    ems.append(lowercase_   )
  lowercase__  :   str    		=      float(sum(lowercase_   ) / len(lowercase_   )   )
  lowercase__  :   str    		=      sum(lowercase_   ) / len(lowercase_   )
  lowercase__  :   List[str]    		=      float(fa_score(y_true=lowercase_ , y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]   )   )
  return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,						_KWARGS_DESCRIPTION							)
class       _snake_case   (	datasets.Metric							):


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name not in [
       "boolq",
       "cb",
       "copa",
       "multirc",
       "record",
       "rte",
       "wic",
       "wsc",
       "wsc.fixed",
       "axb",
       "axg",
   ]:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")
   return datasets.MetricInfo(
       description=_DESCRIPTION					,  citation=_CITATION					,  inputs_description=_KWARGS_DESCRIPTION					,  features=datasets.Features(self._get_feature_types())					,  codebase_urls=[]					,  reference_urls=[]					,  format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None					,  )


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name == "record":
     return {
         "predictions": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "prediction_text": datasets.Value("""string"""),
         },
         "references": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "answers": datasets.Sequence(datasets.Value("""string""")),
         },
     }
   elif self.config_name == "multirc":
     return {
         "predictions": {
             "idx": {
                 "answer": datasets.Value("""int64"""),
                 "paragraph": datasets.Value("""int64"""),
                 "question": datasets.Value("""int64"""),
             },
             "prediction": datasets.Value("""int64"""),
         },
         "references": datasets.Value("""int64"""),
     }
   else:
     return {
         "predictions": datasets.Value("""int64"""),
         "references": datasets.Value("""int64"""),
     }



 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if self.config_name == "axb":
     return {"matthews_correlation": matthews_corrcoef(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   elif self.config_name == "cb":
     return acc_and_fa(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  fa_avg="""macro""")
   elif self.config_name == "record":
     lowercase__  :   Optional[Any]    		=      [
         {
             """qas""": [
                 {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                 for ref in references
             ]
         }
     ]
     lowercase__  :   str    		=      {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
     return evaluate_record(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)[0]
   elif self.config_name == "multirc":
     return evaluate_multirc(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
   elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
     return {"accuracy": simple_accuracy(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   else:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")






