

'''simple docstring'''

from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


A_       						=							"\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

A_       						=							"\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

A_       						=							"\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"





def    A_  ( snake_case      ,       snake_case ):
  return float((preds == labels).mean() )





def    A_  ( snake_case      ,       snake_case      ,       snake_case="binary" ):
  SCREAMING_SNAKE_CASE:Union[str, Any]						      =					simple_accuracy(snake_case      ,       snake_case )
  SCREAMING_SNAKE_CASE:str						      =					float(fa_score(y_true=snake_case      ,       y_pred=snake_case      ,       average=snake_case ) )
  return {
      "accuracy": acc,
      "f1": fa,
  }





def    A_  ( snake_case      ,       snake_case ):
  SCREAMING_SNAKE_CASE:Dict						      =					{}
  for id_pred, label in zip(snake_case      ,       snake_case ):
    SCREAMING_SNAKE_CASE:Union[str, Any]						      =					F'''{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'''
    SCREAMING_SNAKE_CASE:List[str]						      =					id_pred["prediction"]
    if question_id in question_map:
      question_map[question_id].append((pred, label) )
    else:
      SCREAMING_SNAKE_CASE:List[Any]						      =					[(pred, label)]
  SCREAMING_SNAKE_CASE      ,    SCREAMING_SNAKE_CASE:List[str]						      =					[], []
  for question, preds_labels in question_map.items():
    SCREAMING_SNAKE_CASE      ,    SCREAMING_SNAKE_CASE:str						      =					zip(*snake_case )
    SCREAMING_SNAKE_CASE:Optional[Any]						      =					fa_score(y_true=snake_case      ,       y_pred=snake_case      ,       average="macro" )
    fas.append(snake_case )
    SCREAMING_SNAKE_CASE:Dict						      =					int(sum(pred == label for pred, label in preds_labels ) == len(snake_case ) )
    ems.append(snake_case )
  SCREAMING_SNAKE_CASE:Optional[int]						      =					float(sum(snake_case ) / len(snake_case ) )
  SCREAMING_SNAKE_CASE:Optional[int]						      =					sum(snake_case ) / len(snake_case )
  SCREAMING_SNAKE_CASE:Union[str, Any]						      =					float(fa_score(y_true=snake_case      ,       y_pred=[id_pred["prediction"] for id_pred in ids_preds] ) )
  return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,  _KWARGS_DESCRIPTION						)
class 							_snake_case (       datasets.Metric						):
   def     __UpperCamelCase						( self			:      Optional[Any]							):
     if self.config_name not in [
         "boolq",
         "cb",
         "copa",
         "multirc",
         "record",
         "rte",
         "wic",
         "wsc",
         "wsc.fixed",
         "axb",
         "axg",
     ]:
       raise KeyError(
           "You should supply a configuration name selected in "
           "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"							)
     return datasets.MetricInfo(
         description=_DESCRIPTION     ,citation=_CITATION     ,inputs_description=_KWARGS_DESCRIPTION     ,features=datasets.Features(self._get_feature_types()							)     ,codebase_urls=[]     ,reference_urls=[]     ,format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None     ,)
   def     __UpperCamelCase						( self			:      int							):
     if self.config_name == "record":
       return {
           "predictions": {
               "idx": {
                   "passage": datasets.Value("int64"							),
                   "query": datasets.Value("int64"							),
               },
               "prediction_text": datasets.Value("string"							),
           },
           "references": {
               "idx": {
                   "passage": datasets.Value("int64"							),
                   "query": datasets.Value("int64"							),
               },
               "answers": datasets.Sequence(datasets.Value("string"							)							),
           },
       }
     elif self.config_name == "multirc":
       return {
           "predictions": {
               "idx": {
                   "answer": datasets.Value("int64"							),
                   "paragraph": datasets.Value("int64"							),
                   "question": datasets.Value("int64"							),
               },
               "prediction": datasets.Value("int64"							),
           },
           "references": datasets.Value("int64"							),
       }
     else:
       return {
           "predictions": datasets.Value("int64"							),
           "references": datasets.Value("int64"							),
       }







   def     __UpperCamelCase						( self			:      Tuple     ,SCREAMING_SNAKE_CASE__			:      List[str]     ,SCREAMING_SNAKE_CASE__			:      List[Any]							):
     if self.config_name == "axb":
       return {"matthews_correlation": matthews_corrcoef(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)}
     elif self.config_name == "cb":
       return acc_and_fa(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,fa_avg="macro"							)
     elif self.config_name == "record":
       SCREAMING_SNAKE_CASE:str						      =					[
           {
               "qas": [
                   {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
                   for ref in references
               ]
           }
       ]
       SCREAMING_SNAKE_CASE:Union[str, Any]						      =					{pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
       return evaluate_record(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)[0]
     elif self.config_name == "multirc":
       return evaluate_multirc(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)
     elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
       return {"accuracy": simple_accuracy(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)}
     else:
       raise KeyError(
           "You should supply a configuration name selected in "
           "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"							)









'''simple docstring'''

from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


A_       						=							logging.get_logger(__name__)


if is_vision_available():
    import PIL





class 							_snake_case (       _a						):
   _A       :     int       =					['''pixel_values''']
   def __init__( self			:      Optional[Any]     ,SCREAMING_SNAKE_CASE__			:      bool = True     ,SCREAMING_SNAKE_CASE__			:      Dict[str, int] = None     ,SCREAMING_SNAKE_CASE__			:      PILImageResampling = PILImageResampling.BICUBIC     ,SCREAMING_SNAKE_CASE__			:      bool = True     ,SCREAMING_SNAKE_CASE__			:      Dict[str, int] = None     ,SCREAMING_SNAKE_CASE__			:      bool = True     ,SCREAMING_SNAKE_CASE__			:      Union[int, float] = 1 / 255     ,SCREAMING_SNAKE_CASE__			:      bool = True     ,SCREAMING_SNAKE_CASE__			:      Optional[Union[float, List[float]]] = None     ,SCREAMING_SNAKE_CASE__			:      Optional[Union[float, List[float]]] = None     ,SCREAMING_SNAKE_CASE__			:      bool = True     ,**SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,):
     super().__init__(**SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Any						      =					size if size is not None else {"shortest_edge": 224}
     SCREAMING_SNAKE_CASE:int						      =					get_size_dict(SCREAMING_SNAKE_CASE__     ,default_to_square=SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:str						      =					crop_size if crop_size is not None else {"height": 224, "width": 224}
     SCREAMING_SNAKE_CASE:str						      =					get_size_dict(SCREAMING_SNAKE_CASE__     ,default_to_square=SCREAMING_SNAKE_CASE__     ,param_name="crop_size"							)

     SCREAMING_SNAKE_CASE:List[Any]						      =					do_resize
     SCREAMING_SNAKE_CASE:Optional[int]						      =					size
     SCREAMING_SNAKE_CASE:List[Any]						      =					resample
     SCREAMING_SNAKE_CASE:Any						      =					do_center_crop
     SCREAMING_SNAKE_CASE:List[Any]						      =					crop_size
     SCREAMING_SNAKE_CASE:Tuple						      =					do_rescale
     SCREAMING_SNAKE_CASE:Optional[Any]						      =					rescale_factor
     SCREAMING_SNAKE_CASE:Dict						      =					do_normalize
     SCREAMING_SNAKE_CASE:int						      =					image_mean if image_mean is not None else OPENAI_CLIP_MEAN
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					image_std if image_std is not None else OPENAI_CLIP_STD
     SCREAMING_SNAKE_CASE:Optional[int]						      =					do_convert_rgb
   def     __UpperCamelCase						( self			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      np.ndarray     ,SCREAMING_SNAKE_CASE__			:      Dict[str, int]     ,SCREAMING_SNAKE_CASE__			:      PILImageResampling = PILImageResampling.BICUBIC     ,SCREAMING_SNAKE_CASE__			:      Optional[Union[str, ChannelDimension]] = None     ,**SCREAMING_SNAKE_CASE__			:      Any     ,):
     SCREAMING_SNAKE_CASE:List[Any]						      =					get_size_dict(SCREAMING_SNAKE_CASE__     ,default_to_square=SCREAMING_SNAKE_CASE__							)
     if "shortest_edge" not in size:
       raise ValueError(F'''The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'''							)
     SCREAMING_SNAKE_CASE:Dict						      =					get_resize_output_image_size(SCREAMING_SNAKE_CASE__     ,size=size["shortest_edge"]     ,default_to_square=SCREAMING_SNAKE_CASE__							)
     return resize(SCREAMING_SNAKE_CASE__     ,size=SCREAMING_SNAKE_CASE__     ,resample=SCREAMING_SNAKE_CASE__     ,data_format=SCREAMING_SNAKE_CASE__     ,**SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      np.ndarray     ,SCREAMING_SNAKE_CASE__			:      Dict[str, int]     ,SCREAMING_SNAKE_CASE__			:      Optional[Union[str, ChannelDimension]] = None     ,**SCREAMING_SNAKE_CASE__			:      Optional[int]     ,):
     SCREAMING_SNAKE_CASE:List[Any]						      =					get_size_dict(SCREAMING_SNAKE_CASE__							)
     if "height" not in size or "width" not in size:
       raise ValueError(F'''The `size` parameter must contain the keys (height, width). Got {size.keys()}'''							)
     return center_crop(SCREAMING_SNAKE_CASE__     ,size=(size["height"], size["width"])     ,data_format=SCREAMING_SNAKE_CASE__     ,**SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      Dict     ,SCREAMING_SNAKE_CASE__			:      np.ndarray     ,SCREAMING_SNAKE_CASE__			:      Union[int, float]     ,SCREAMING_SNAKE_CASE__			:      Optional[Union[str, ChannelDimension]] = None     ,**SCREAMING_SNAKE_CASE__			:      List[Any]     ,):
     return rescale(SCREAMING_SNAKE_CASE__     ,scale=SCREAMING_SNAKE_CASE__     ,data_format=SCREAMING_SNAKE_CASE__     ,**SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      Any     ,SCREAMING_SNAKE_CASE__			:      np.ndarray     ,SCREAMING_SNAKE_CASE__			:      Union[float, List[float]]     ,SCREAMING_SNAKE_CASE__			:      Union[float, List[float]]     ,SCREAMING_SNAKE_CASE__			:      Optional[Union[str, ChannelDimension]] = None     ,**SCREAMING_SNAKE_CASE__			:      Tuple     ,):
     return normalize(SCREAMING_SNAKE_CASE__     ,mean=SCREAMING_SNAKE_CASE__     ,std=SCREAMING_SNAKE_CASE__     ,data_format=SCREAMING_SNAKE_CASE__     ,**SCREAMING_SNAKE_CASE__							)







   def     __UpperCamelCase						( self			:      int     ,SCREAMING_SNAKE_CASE__			:      ImageInput     ,SCREAMING_SNAKE_CASE__			:      bool = None     ,SCREAMING_SNAKE_CASE__			:      Dict[str, int] = None     ,SCREAMING_SNAKE_CASE__			:      PILImageResampling = None     ,SCREAMING_SNAKE_CASE__			:      bool = None     ,SCREAMING_SNAKE_CASE__			:      int = None     ,SCREAMING_SNAKE_CASE__			:      bool = None     ,SCREAMING_SNAKE_CASE__			:      float = None     ,SCREAMING_SNAKE_CASE__			:      bool = None     ,SCREAMING_SNAKE_CASE__			:      Optional[Union[float, List[float]]] = None     ,SCREAMING_SNAKE_CASE__			:      Optional[Union[float, List[float]]] = None     ,SCREAMING_SNAKE_CASE__			:      bool = None     ,SCREAMING_SNAKE_CASE__			:      Optional[Union[str, TensorType]] = None     ,SCREAMING_SNAKE_CASE__			:      Optional[ChannelDimension] = ChannelDimension.FIRST     ,**SCREAMING_SNAKE_CASE__			:      List[str]     ,):
     SCREAMING_SNAKE_CASE:Any						      =					do_resize if do_resize is not None else self.do_resize
     SCREAMING_SNAKE_CASE:Optional[int]						      =					size if size is not None else self.size
     SCREAMING_SNAKE_CASE:Dict						      =					get_size_dict(SCREAMING_SNAKE_CASE__     ,param_name="size"     ,default_to_square=SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Optional[int]						      =					resample if resample is not None else self.resample
     SCREAMING_SNAKE_CASE:Dict						      =					do_center_crop if do_center_crop is not None else self.do_center_crop
     SCREAMING_SNAKE_CASE:Optional[int]						      =					crop_size if crop_size is not None else self.crop_size
     SCREAMING_SNAKE_CASE:str						      =					get_size_dict(SCREAMING_SNAKE_CASE__     ,param_name="crop_size"     ,default_to_square=SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:int						      =					do_rescale if do_rescale is not None else self.do_rescale
     SCREAMING_SNAKE_CASE:Tuple						      =					rescale_factor if rescale_factor is not None else self.rescale_factor
     SCREAMING_SNAKE_CASE:List[Any]						      =					do_normalize if do_normalize is not None else self.do_normalize
     SCREAMING_SNAKE_CASE:List[str]						      =					image_mean if image_mean is not None else self.image_mean
     SCREAMING_SNAKE_CASE:Any						      =					image_std if image_std is not None else self.image_std
     SCREAMING_SNAKE_CASE:str						      =					do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

     SCREAMING_SNAKE_CASE:Any						      =					make_list_of_images(SCREAMING_SNAKE_CASE__							)

     if not valid_images(SCREAMING_SNAKE_CASE__							):
       raise ValueError(
           "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
           "torch.Tensor, tf.Tensor or jax.ndarray."							)

     if do_resize and size is None:
       raise ValueError("Size must be specified if do_resize is True."							)

     if do_center_crop and crop_size is None:
       raise ValueError("Crop size must be specified if do_center_crop is True."							)

     if do_rescale and rescale_factor is None:
       raise ValueError("Rescale factor must be specified if do_rescale is True."							)

     if do_normalize and (image_mean is None or image_std is None):
       raise ValueError("Image mean and std must be specified if do_normalize is True."							)

     # PIL RGBA images are converted to RGB
     if do_convert_rgb:
       SCREAMING_SNAKE_CASE:List[Any]						      =					[convert_to_rgb(SCREAMING_SNAKE_CASE__							) for image in images]

     # All transformations expect numpy arrays.
     SCREAMING_SNAKE_CASE:Optional[int]						      =					[to_numpy_array(SCREAMING_SNAKE_CASE__							) for image in images]

     if do_resize:
       SCREAMING_SNAKE_CASE:Union[str, Any]						      =					[self.resize(image=SCREAMING_SNAKE_CASE__     ,size=SCREAMING_SNAKE_CASE__     ,resample=SCREAMING_SNAKE_CASE__							) for image in images]

     if do_center_crop:
       SCREAMING_SNAKE_CASE:Union[str, Any]						      =					[self.center_crop(image=SCREAMING_SNAKE_CASE__     ,size=SCREAMING_SNAKE_CASE__							) for image in images]

     if do_rescale:
       SCREAMING_SNAKE_CASE:Any						      =					[self.rescale(image=SCREAMING_SNAKE_CASE__     ,scale=SCREAMING_SNAKE_CASE__							) for image in images]

     if do_normalize:
       SCREAMING_SNAKE_CASE:List[str]						      =					[self.normalize(image=SCREAMING_SNAKE_CASE__     ,mean=SCREAMING_SNAKE_CASE__     ,std=SCREAMING_SNAKE_CASE__							) for image in images]

     SCREAMING_SNAKE_CASE:List[Any]						      =					[to_channel_dimension_format(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							) for image in images]

     SCREAMING_SNAKE_CASE:Optional[int]						      =					{"pixel_values": images}
     return BatchFeature(data=SCREAMING_SNAKE_CASE__     ,tensor_type=SCREAMING_SNAKE_CASE__							)






