



"""simple docstring"""

from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowercase__					:						Tuple												=			'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

lowercase__					:						Optional[Any]												=			'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

lowercase__					:						str												=			'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''




def __lowercase    (							_a	,    _a					):
    return float((preds == labels).mean()					)




def __lowercase    (							_a	,    _a	,    _a="binary"					):
    snake_case_				:  List[Any]       =			simple_accuracy(_a	,    _a					)
    snake_case_				:  List[str]       =			float(fa_score(y_true=_a	,    y_pred=_a	,    average=_a					)					)
    return {
        "accuracy": acc,
        "f1": fa,
    }




def __lowercase    (							_a	,    _a					):
    snake_case_				:  Any       =			{}
    for id_pred, label in zip(_a	,    _a					):
        snake_case_				:  Union[str, Any]       =			f"{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"
        snake_case_				:  int       =			id_pred['''prediction''']
        if question_id in question_map:
            question_map[question_id].append((pred, label)					)
        else:
            snake_case_				:  List[str]       =			[(pred, label)]
    snake_case_,			snake_case_				:  str       =			[], []
    for question, preds_labels in question_map.items():
        snake_case_,			snake_case_				:  List[Any]       =			zip(*_a					)
        snake_case_				:  List[Any]       =			fa_score(y_true=_a	,    y_pred=_a	,    average='''macro'''					)
        fas.append(_a					)
        snake_case_				:  int       =			int(sum(pred == label for pred, label in preds_labels					) == len(_a					)					)
        ems.append(_a					)
    snake_case_				:  Union[str, Any]       =			float(sum(_a					) / len(_a					)					)
    snake_case_				:  str       =			sum(_a					) / len(_a					)
    snake_case_				:  Union[str, Any]       =			float(fa_score(y_true=_a	,    y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]					)					)
    return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,  _KWARGS_DESCRIPTION)
class  _UpperCAmelCase (    datasets.Metric):

 def 		_snake_case      (  self	: List[Any]       ):
     if self.config_name not in [
         "boolq",
         "cb",
         "copa",
         "multirc",
         "record",
         "rte",
         "wic",
         "wsc",
         "wsc.fixed",
         "axb",
         "axg",
     ]:
         raise KeyError(
             '''You should supply a configuration name selected in '''
             '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''       )
     return datasets.MetricInfo(
         description=_DESCRIPTION						,  citation=_CITATION						,  inputs_description=_KWARGS_DESCRIPTION						,  features=datasets.Features(self._get_feature_types()       )						,  codebase_urls=[]						,  reference_urls=[]						,  format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None						,  )

 def 		_snake_case      (  self	: Optional[Any]       ):
     if self.config_name == "record":
         return {
             "predictions": {
                 "idx": {
                     "passage": datasets.Value('''int64'''       ),
                     "query": datasets.Value('''int64'''       ),
                 },
                 "prediction_text": datasets.Value('''string'''       ),
             },
             "references": {
                 "idx": {
                     "passage": datasets.Value('''int64'''       ),
                     "query": datasets.Value('''int64'''       ),
                 },
                 "answers": datasets.Sequence(datasets.Value('''string'''       )       ),
             },
         }
     elif self.config_name == "multirc":
         return {
             "predictions": {
                 "idx": {
                     "answer": datasets.Value('''int64'''       ),
                     "paragraph": datasets.Value('''int64'''       ),
                     "question": datasets.Value('''int64'''       ),
                 },
                 "prediction": datasets.Value('''int64'''       ),
             },
             "references": datasets.Value('''int64'''       ),
         }
     else:
         return {
             "predictions": datasets.Value('''int64'''       ),
             "references": datasets.Value('''int64'''       ),
         }






 def 		_snake_case      (  self	: Optional[int]						,  lowercase_	: Dict						,  lowercase_	: Any       ):
     if self.config_name == "axb":
         return {"matthews_correlation": matthews_corrcoef(lowercase_						,  lowercase_       )}
     elif self.config_name == "cb":
         return acc_and_fa(lowercase_						,  lowercase_						,  fa_avg='''macro'''       )
     elif self.config_name == "record":
         snake_case_				:  Tuple       =			[
             {
                 '''qas''': [
                     {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
                     for ref in references
                 ]
             }
         ]
         snake_case_				:  Union[str, Any]       =			{pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
         return evaluate_record(lowercase_						,  lowercase_       )[0]
     elif self.config_name == "multirc":
         return evaluate_multirc(lowercase_						,  lowercase_       )
     elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
         return {"accuracy": simple_accuracy(lowercase_						,  lowercase_       )}
     else:
         raise KeyError(
             '''You should supply a configuration name selected in '''
             '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''       )










"""simple docstring"""

from __future__ import annotations

import inspect
import unittest
from math import floor

import numpy as np

from transformers import CvtConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import TFCvtForImageClassification, TFCvtModel
     from transformers.models.cvt.modeling_tf_cvt import TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
     from PIL import Image

     from transformers import AutoImageProcessor




class  _UpperCAmelCase (    lowerCAmelCase__):

 def 		_snake_case      (  self	: int       ):
     snake_case_				:  Any       =			self.config_class(**self.inputs_dict       )
     self.parent.assertTrue(hasattr(lowercase_						,  '''embed_dim'''       )       )
     self.parent.assertTrue(hasattr(lowercase_						,  '''num_heads'''       )       )





class  _UpperCAmelCase :

 def __init__(  self	: List[str]						,  lowercase_	: Optional[Any]						,  lowercase_	: int=13						,  lowercase_	: Optional[int]=64						,  lowercase_	: Any=3						,  lowercase_	: Any=[16, 48, 96]						,  lowercase_	: List[Any]=[1, 3, 6]						,  lowercase_	: Union[str, Any]=[1, 2, 10]						,  lowercase_	: Optional[Any]=[7, 3, 3]						,  lowercase_	: Union[str, Any]=[4, 2, 2]						,  lowercase_	: Tuple=[2, 1, 1]						,  lowercase_	: List[str]=[2, 2, 2]						,  lowercase_	: Union[str, Any]=[False, False, True]						,  lowercase_	: Optional[int]=[0.0, 0.0, 0.0]						,  lowercase_	: str=0.02						,  lowercase_	: Optional[Any]=1E-12						,  lowercase_	: Optional[int]=True						,  lowercase_	: Optional[int]=True						,  lowercase_	: Optional[Any]=2						,  ):
     snake_case_				:  List[Any]       =			parent
     snake_case_				:  int       =			batch_size
     snake_case_				:  Union[str, Any]       =			image_size
     snake_case_				:  Tuple       =			patch_sizes
     snake_case_				:  List[Any]       =			patch_stride
     snake_case_				:  Dict       =			patch_padding
     snake_case_				:  Any       =			is_training
     snake_case_				:  Any       =			use_labels
     snake_case_				:  str       =			num_labels
     snake_case_				:  Optional[Any]       =			num_channels
     snake_case_				:  Optional[Any]       =			embed_dim
     snake_case_				:  int       =			num_heads
     snake_case_				:  List[str]       =			stride_kv
     snake_case_				:  Any       =			depth
     snake_case_				:  Dict       =			cls_token
     snake_case_				:  Dict       =			attention_drop_rate
     snake_case_				:  int       =			initializer_range
     snake_case_				:  Tuple       =			layer_norm_eps

 def 		_snake_case      (  self	: Dict       ):
     snake_case_				:  str       =			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]       )

     snake_case_				:  str       =			None
     if self.use_labels:
         # create a random int32 tensor of given shape
         snake_case_				:  Union[str, Any]       =			ids_tensor([self.batch_size]						,  self.num_labels       )

     snake_case_				:  Dict       =			self.get_config()
     return config, pixel_values, labels

 def 		_snake_case      (  self	: int       ):
     return CvtConfig(
         image_size=self.image_size						,  num_labels=self.num_labels						,  num_channels=self.num_channels						,  embed_dim=self.embed_dim						,  num_heads=self.num_heads						,  patch_sizes=self.patch_sizes						,  patch_padding=self.patch_padding						,  patch_stride=self.patch_stride						,  stride_kv=self.stride_kv						,  depth=self.depth						,  cls_token=self.cls_token						,  attention_drop_rate=self.attention_drop_rate						,  initializer_range=self.initializer_range						,  )

 def 		_snake_case      (  self	: int						,  lowercase_	: List[str]						,  lowercase_	: Optional[Any]						,  lowercase_	: Union[str, Any]       ):
     snake_case_				:  Tuple       =			TFCvtModel(config=lowercase_       )
     snake_case_				:  Tuple       =			model(lowercase_						,  training=lowercase_       )
     snake_case_				:  int       =			(self.image_size, self.image_size)
     snake_case_,			snake_case_				:  List[str]       =			image_size[0], image_size[1]
     for i in range(len(self.depth       )       ):
         snake_case_				:  str       =			floor(((height + 2 * self.patch_padding[i] - self.patch_sizes[i]) / self.patch_stride[i]) + 1       )
         snake_case_				:  str       =			floor(((width + 2 * self.patch_padding[i] - self.patch_sizes[i]) / self.patch_stride[i]) + 1       )
     self.parent.assertEqual(result.last_hidden_state.shape						,  (self.batch_size, self.embed_dim[-1], height, width)       )

 def 		_snake_case      (  self	: Dict						,  lowercase_	: Tuple						,  lowercase_	: Union[str, Any]						,  lowercase_	: List[str]       ):
     snake_case_				:  int       =			self.num_labels
     snake_case_				:  Any       =			TFCvtForImageClassification(lowercase_       )
     snake_case_				:  List[Any]       =			model(lowercase_						,  labels=lowercase_						,  training=lowercase_       )
     self.parent.assertEqual(result.logits.shape						,  (self.batch_size, self.num_labels)       )






 def 		_snake_case      (  self	: Any       ):
     snake_case_				:  Tuple       =			self.prepare_config_and_inputs()
     snake_case_,			snake_case_,			snake_case_				:  List[str]       =			config_and_inputs
     snake_case_				:  Any       =			{'''pixel_values''': pixel_values}
     return config, inputs_dict





@require_tf
class  _UpperCAmelCase (    lowerCAmelCase__					,  lowerCAmelCase__					,  unittest.TestCase):
 _lowerCAmelCase     :    Optional[int]	      =	(TFCvtModel, TFCvtForImageClassification) if is_tf_available() else ()
 _lowerCAmelCase     :    str	      =	(
     {"""feature-extraction""": TFCvtModel, """image-classification""": TFCvtForImageClassification}
     if is_tf_available()
     else {}
 )
 _lowerCAmelCase     :    str	      =	False
 _lowerCAmelCase     :    int	      =	False
 _lowerCAmelCase     :    Tuple	      =	False
 _lowerCAmelCase     :    int	      =	False
 _lowerCAmelCase     :    int	      =	False

 def 		_snake_case      (  self	: int       ):
     snake_case_				:  Optional[int]       =			TFCvtModelTester(self       )
     snake_case_				:  str       =			TFCvtConfigTester(self						,  config_class=lowercase_						,  has_text_modality=lowercase_						,  hidden_size=37       )

 def 		_snake_case      (  self	: int       ):
     self.config_tester.create_and_test_config_common_properties()
     self.config_tester.create_and_test_config_to_json_string()
     self.config_tester.create_and_test_config_to_json_file()
     self.config_tester.create_and_test_config_from_and_save_pretrained()
     self.config_tester.create_and_test_config_with_num_labels()
     self.config_tester.check_config_can_be_init_without_params()
     self.config_tester.check_config_arguments_init()

 @unittest.skip(reason='''Cvt does not output attentions'''       )
 def 		_snake_case      (  self	: Any       ):
     pass

 @unittest.skip(reason='''Cvt does not use inputs_embeds'''       )
 def 		_snake_case      (  self	: str       ):
     pass

 @unittest.skip(reason='''Cvt does not support input and output embeddings'''       )
 def 		_snake_case      (  self	: Union[str, Any]       ):
     pass

 @unittest.skipIf(
     not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''       )       ) == 0						,  reason='''TF does not support backprop for grouped convolutions on CPU.'''						,  )
 def 		_snake_case      (  self	: Tuple       ):
     super().test_dataset_conversion()

 @unittest.skipIf(
     not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''       )       ) == 0						,  reason='''TF does not support backprop for grouped convolutions on CPU.'''						,  )
 @slow
 def 		_snake_case      (  self	: Tuple       ):
     super().test_keras_fit()

 @unittest.skip(reason='''Get `Failed to determine best cudnn convolution algo.` error after using TF 2.12+cuda 11.8'''       )
 def 		_snake_case      (  self	: List[str]       ):
     snake_case_				:  Optional[Any]       =			tf.keras.mixed_precision.Policy('''mixed_float16'''       )
     tf.keras.mixed_precision.set_global_policy(lowercase_       )
     super().test_keras_fit()
     tf.keras.mixed_precision.set_global_policy('''float32'''       )

 def 		_snake_case      (  self	: int       ):
     snake_case_,			snake_case_				:  List[str]       =			self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
         snake_case_				:  Dict       =			model_class(lowercase_       )
         snake_case_				:  str       =			inspect.signature(model.call       )
         # signature.parameters is an OrderedDict => so arg_names order is deterministic
         snake_case_				:  List[str]       =			[*signature.parameters.keys()]

         snake_case_				:  int       =			['''pixel_values''']
         self.assertListEqual(arg_names[:1]						,  lowercase_       )

 def 		_snake_case      (  self	: List[str]       ):
     def check_hidden_states_output(lowercase_	: Optional[int]						,  lowercase_	: Union[str, Any]						,  lowercase_	: str       ):
         snake_case_				:  Any       =			model_class(lowercase_       )

         snake_case_				:  Optional[Any]       =			model(**self._prepare_for_class(lowercase_						,  lowercase_       )       )
         snake_case_				:  Tuple       =			outputs.hidden_states

         snake_case_				:  str       =			len(self.model_tester.depth       )
         self.assertEqual(len(lowercase_       )						,  lowercase_       )

         # verify the first hidden states (first block)
         self.assertListEqual(
             list(hidden_states[0].shape[-3:]       )						,  [
                 self.model_tester.embed_dim[0],
                 self.model_tester.image_size // 4,
                 self.model_tester.image_size // 4,
             ]						,  )

     snake_case_,			snake_case_				:  List[Any]       =			self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
         snake_case_				:  Dict       =			True
         check_hidden_states_output(lowercase_						,  lowercase_						,  lowercase_       )

         # check that output_hidden_states also work using config
         del inputs_dict["output_hidden_states"]
         snake_case_				:  Tuple       =			True

         check_hidden_states_output(lowercase_						,  lowercase_						,  lowercase_       )

 def 		_snake_case      (  self	: str       ):
     snake_case_				:  Dict       =			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*lowercase_       )

 def 		_snake_case      (  self	: List[Any]       ):
     snake_case_				:  str       =			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_image_classification(*lowercase_       )






 @slow
 def 		_snake_case      (  self	: Optional[Any]       ):
     for model_name in TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         snake_case_				:  int       =			TFCvtModel.from_pretrained(lowercase_       )
         self.assertIsNotNone(lowercase_       )





def __lowercase    (							):
    snake_case_				:  Tuple       =			Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''					)
    return image






@require_tf
@require_vision
class  _UpperCAmelCase (    unittest.TestCase):

 @cached_property
 def 		_snake_case      (  self	: List[Any]       ):
     return AutoImageProcessor.from_pretrained(TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST[0]       )






 @slow
 def 		_snake_case      (  self	: Tuple       ):
     snake_case_				:  int       =			TFCvtForImageClassification.from_pretrained(TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST[0]       )

     snake_case_				:  Any       =			self.default_image_processor
     snake_case_				:  Union[str, Any]       =			prepare_img()
     snake_case_				:  int       =			image_processor(images=lowercase_						,  return_tensors='''tf'''       )

     # forward pass
     snake_case_				:  Tuple       =			model(**lowercase_       )

     # verify the logits
     snake_case_				:  Optional[Any]       =			tf.TensorShape((1, 1000)       )
     self.assertEqual(outputs.logits.shape						,  lowercase_       )

     snake_case_				:  Tuple       =			tf.constant([0.92_85, 0.90_15, -0.31_50]       )
     self.assertTrue(np.allclose(outputs.logits[0, :3].numpy()						,  lowercase_						,  atol=1E-4       )       )





