
from __future__ import annotations

import inspect
import unittest
from math import floor

import numpy as np

from transformers import CvtConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import TFCvtForImageClassification, TFCvtModel
					from transformers.models.cvt.modeling_tf_cvt import TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
					from PIL import Image

					from transformers import AutoImageProcessor



class       _snake_case   (	UpperCAmelCase_							):


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Union[str, Any]    		=      self.config_class(**self.inputs_dict)
			self.parent.assertTrue(hasattr(SCREAMING_SNAKE_CASE_					,  """embed_dim"""))
			self.parent.assertTrue(hasattr(SCREAMING_SNAKE_CASE_					,  """num_heads"""))



class       _snake_case   :


	def __init__(					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_=13					,  SCREAMING_SNAKE_CASE_=64					,  SCREAMING_SNAKE_CASE_=3					,  SCREAMING_SNAKE_CASE_=[16, 48, 96]					,  SCREAMING_SNAKE_CASE_=[1, 3, 6]					,  SCREAMING_SNAKE_CASE_=[1, 2, 10]					,  SCREAMING_SNAKE_CASE_=[7, 3, 3]					,  SCREAMING_SNAKE_CASE_=[4, 2, 2]					,  SCREAMING_SNAKE_CASE_=[2, 1, 1]					,  SCREAMING_SNAKE_CASE_=[2, 2, 2]					,  SCREAMING_SNAKE_CASE_=[False, False, True]					,  SCREAMING_SNAKE_CASE_=[0.0, 0.0, 0.0]					,  SCREAMING_SNAKE_CASE_=0.0_2					,  SCREAMING_SNAKE_CASE_=1E-12					,  SCREAMING_SNAKE_CASE_=True					,  SCREAMING_SNAKE_CASE_=True					,  SCREAMING_SNAKE_CASE_=2					,  ):


			'''simple docstring'''




			lowercase__  :   str    		=      parent
			lowercase__  :   Dict    		=      batch_size
			lowercase__  :   int    		=      image_size
			lowercase__  :   Union[str, Any]    		=      patch_sizes
			lowercase__  :   Union[str, Any]    		=      patch_stride
			lowercase__  :   Optional[Any]    		=      patch_padding
			lowercase__  :   Tuple    		=      is_training
			lowercase__  :   Union[str, Any]    		=      use_labels
			lowercase__  :   int    		=      num_labels
			lowercase__  :   int    		=      num_channels
			lowercase__  :   List[Any]    		=      embed_dim
			lowercase__  :   List[str]    		=      num_heads
			lowercase__  :   Optional[Any]    		=      stride_kv
			lowercase__  :   Union[str, Any]    		=      depth
			lowercase__  :   Optional[int]    		=      cls_token
			lowercase__  :   List[str]    		=      attention_drop_rate
			lowercase__  :   Any    		=      initializer_range
			lowercase__  :   Optional[Any]    		=      layer_norm_eps


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Any    		=      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

			lowercase__  :   Union[str, Any]    		=      None
			if self.use_labels:
					# create a random int32 tensor of given shape
					lowercase__  :   str    		=      ids_tensor([self.batch_size]					,  self.num_labels)

			lowercase__  :   List[str]    		=      self.get_config()
			return config, pixel_values, labels


	def      lowercase__     (					self):


			'''simple docstring'''




			return CvtConfig(
			    image_size=self.image_size					,  num_labels=self.num_labels					,  num_channels=self.num_channels					,  embed_dim=self.embed_dim					,  num_heads=self.num_heads					,  patch_sizes=self.patch_sizes					,  patch_padding=self.patch_padding					,  patch_stride=self.patch_stride					,  stride_kv=self.stride_kv					,  depth=self.depth					,  cls_token=self.cls_token					,  attention_drop_rate=self.attention_drop_rate					,  initializer_range=self.initializer_range					,  )


	def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


			'''simple docstring'''




			lowercase__  :   Any    		=      TFCvtModel(config=SCREAMING_SNAKE_CASE_)
			lowercase__  :   Optional[int]    		=      model(SCREAMING_SNAKE_CASE_					,  training=SCREAMING_SNAKE_CASE_)
			lowercase__  :   str    		=      (self.image_size, self.image_size)
			lowercase__  :   Dict    		=      image_size[0], image_size[1]
			for i in range(len(self.depth)):
					lowercase__  :   str    		=      floor(((height + 2 * self.patch_padding[i] - self.patch_sizes[i]) / self.patch_stride[i]) + 1)
					lowercase__  :   List[Any]    		=      floor(((width + 2 * self.patch_padding[i] - self.patch_sizes[i]) / self.patch_stride[i]) + 1)
			self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.embed_dim[-1], height, width))


	def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


			'''simple docstring'''




			lowercase__  :   int    		=      self.num_labels
			lowercase__  :   Union[str, Any]    		=      TFCvtForImageClassification(SCREAMING_SNAKE_CASE_)
			lowercase__  :   Optional[int]    		=      model(SCREAMING_SNAKE_CASE_					,  labels=SCREAMING_SNAKE_CASE_					,  training=SCREAMING_SNAKE_CASE_)
			self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_labels))



	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Optional[int]    		=      self.prepare_config_and_inputs()
			lowercase__  :   Any    		=      config_and_inputs
			lowercase__  :   List[str]    		=      {"""pixel_values""": pixel_values}
			return config, inputs_dict



@require_tf
class       _snake_case   (	UpperCAmelCase_  ,						UpperCAmelCase_  ,						unittest.TestCase							):
	__lowerCAmelCase  :     List[Any]        =			(TFCvtModel, TFCvtForImageClassification) if is_tf_available() else ()
	__lowerCAmelCase  :     Optional[Any]        =			(
	    {'feature-extraction': TFCvtModel, 'image-classification': TFCvtForImageClassification}
	    if is_tf_available()
	    else {}
	)
	__lowerCAmelCase  :     List[Any]        =			False
	__lowerCAmelCase  :     Dict        =			False
	__lowerCAmelCase  :     List[str]        =			False
	__lowerCAmelCase  :     Any        =			False
	__lowerCAmelCase  :     str        =			False


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Optional[Any]    		=      TFCvtModelTester(self)
			lowercase__  :   Optional[Any]    		=      TFCvtConfigTester(self					,  config_class=SCREAMING_SNAKE_CASE_					,  has_text_modality=SCREAMING_SNAKE_CASE_					,  hidden_size=37)


	def      lowercase__     (					self):


			'''simple docstring'''




			self.config_tester.create_and_test_config_common_properties()
			self.config_tester.create_and_test_config_to_json_string()
			self.config_tester.create_and_test_config_to_json_file()
			self.config_tester.create_and_test_config_from_and_save_pretrained()
			self.config_tester.create_and_test_config_with_num_labels()
			self.config_tester.check_config_can_be_init_without_params()
			self.config_tester.check_config_arguments_init()


	@unittest.skip(reason="""Cvt does not output attentions""")
	def      lowercase__     (					self):


			'''simple docstring'''




			pass


	@unittest.skip(reason="""Cvt does not use inputs_embeds""")
	def      lowercase__     (					self):


			'''simple docstring'''




			pass


	@unittest.skip(reason="""Cvt does not support input and output embeddings""")
	def      lowercase__     (					self):


			'''simple docstring'''




			pass


	@unittest.skipIf(
	    not is_tf_available() or len(tf.config.list_physical_devices("""GPU""")) == 0					,  reason="""TF does not support backprop for grouped convolutions on CPU."""					,  )
	def      lowercase__     (					self):


			'''simple docstring'''




			super().test_dataset_conversion()


	@unittest.skipIf(
	    not is_tf_available() or len(tf.config.list_physical_devices("""GPU""")) == 0					,  reason="""TF does not support backprop for grouped convolutions on CPU."""					,  )
	@slow
	def      lowercase__     (					self):


			'''simple docstring'''




			super().test_keras_fit()


	@unittest.skip(reason="""Get `Failed to determine best cudnn convolution algo.` error after using TF 2.12+cuda 11.8""")
	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   List[Any]    		=      tf.keras.mixed_precision.Policy("""mixed_float16""")
			tf.keras.mixed_precision.set_global_policy(SCREAMING_SNAKE_CASE_)
			super().test_keras_fit()
			tf.keras.mixed_precision.set_global_policy("""float32""")


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Dict    		=      self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					lowercase__  :   Union[str, Any]    		=      model_class(SCREAMING_SNAKE_CASE_)
					lowercase__  :   List[str]    		=      inspect.signature(model.call)
					# signature.parameters is an OrderedDict => so arg_names order is deterministic
					lowercase__  :   Any    		=      [*signature.parameters.keys()]

					lowercase__  :   Dict    		=      ["""pixel_values"""]
					self.assertListEqual(arg_names[:1]					,  SCREAMING_SNAKE_CASE_)


	def      lowercase__     (					self):


			'''simple docstring'''




			def check_hidden_states_output(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):
					lowercase__  :   Any    		=      model_class(SCREAMING_SNAKE_CASE_)

					lowercase__  :   Dict    		=      model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_))
					lowercase__  :   List[Any]    		=      outputs.hidden_states

					lowercase__  :   Union[str, Any]    		=      len(self.model_tester.depth)
					self.assertEqual(len(SCREAMING_SNAKE_CASE_)					,  SCREAMING_SNAKE_CASE_)

					# verify the first hidden states (first block)
					self.assertListEqual(
					    list(hidden_states[0].shape[-3:])					,  [
					        self.model_tester.embed_dim[0],
					        self.model_tester.image_size // 4,
					        self.model_tester.image_size // 4,
					    ]					,  )

			lowercase__  :   Optional[Any]    		=      self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					lowercase__  :   Any    		=      True
					check_hidden_states_output(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)

					# check that output_hidden_states also work using config
					del inputs_dict["output_hidden_states"]
					lowercase__  :   int    		=      True

					check_hidden_states_output(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   List[Any]    		=      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_)


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Tuple    		=      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_image_classification(*SCREAMING_SNAKE_CASE_)



	@slow
	def      lowercase__     (					self):


			'''simple docstring'''




			for model_name in TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					lowercase__  :   Optional[int]    		=      TFCvtModel.from_pretrained(SCREAMING_SNAKE_CASE_)
					self.assertIsNotNone(SCREAMING_SNAKE_CASE_)

def 		UpperCamelCase	(      )		->	Optional[Any]:


		'''simple docstring'''






		lowercase__  :   Union[str, Any]    		=      Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""   )
		return image



@require_tf
@require_vision
class       _snake_case   (	unittest.TestCase							):


	@cached_property
	def      lowercase__     (					self):


			'''simple docstring'''




			return AutoImageProcessor.from_pretrained(TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST[0])



	@slow
	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Any    		=      TFCvtForImageClassification.from_pretrained(TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST[0])

			lowercase__  :   int    		=      self.default_image_processor
			lowercase__  :   Optional[Any]    		=      prepare_img()
			lowercase__  :   Union[str, Any]    		=      image_processor(images=SCREAMING_SNAKE_CASE_					,  return_tensors="""tf""")

			# forward pass
			lowercase__  :   Any    		=      model(**SCREAMING_SNAKE_CASE_)

			# verify the logits
			lowercase__  :   Any    		=      tf.TensorShape((1, 10_00))
			self.assertEqual(outputs.logits.shape					,  SCREAMING_SNAKE_CASE_)

			lowercase__  :   List[Any]    		=      tf.constant([0.9_2_8_5, 0.9_0_1_5, -0.3_1_5_0])
			self.assertTrue(np.allclose(outputs.logits[0, :3].numpy()					,  SCREAMING_SNAKE_CASE_					,  atol=1E-4))








from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowerCamelCase__    :		Dict          =   """\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

lowerCamelCase__    :		Tuple          =   """\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

lowerCamelCase__    :		Any          =   """
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Optional[Any]:


  '''simple docstring'''






  return float((preds == labels).mean()   )

def 		UpperCamelCase	(      lowercase_ , lowercase_ , lowercase_="binary"   )		->	int:


  '''simple docstring'''






  lowercase__  :   List[str]    		=      simple_accuracy(lowercase_ , lowercase_   )
  lowercase__  :   Optional[Any]    		=      float(fa_score(y_true=lowercase_ , y_pred=lowercase_ , average=lowercase_   )   )
  return {
      "accuracy": acc,
      "f1": fa,
  }

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Dict:


  '''simple docstring'''






  lowercase__  :   List[Any]    		=      {}
  for id_pred, label in zip(lowercase_ , lowercase_   ):
    lowercase__  :   Optional[Any]    		=      F'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
    lowercase__  :   Optional[Any]    		=      id_pred["""prediction"""]
    if question_id in question_map:
      question_map[question_id].append((pred, label)   )
    else:
      lowercase__  :   Optional[int]    		=      [(pred, label)]
  lowercase__		,    lowercase__  :   List[str]    		=      [], []
  for question, preds_labels in question_map.items():
    lowercase__		,    lowercase__  :   List[str]    		=      zip(*lowercase_   )
    lowercase__  :   Optional[Any]    		=      fa_score(y_true=lowercase_ , y_pred=lowercase_ , average="""macro"""   )
    fas.append(lowercase_   )
    lowercase__  :   List[Any]    		=      int(sum(pred == label for pred, label in preds_labels   ) == len(lowercase_   )   )
    ems.append(lowercase_   )
  lowercase__  :   str    		=      float(sum(lowercase_   ) / len(lowercase_   )   )
  lowercase__  :   str    		=      sum(lowercase_   ) / len(lowercase_   )
  lowercase__  :   List[str]    		=      float(fa_score(y_true=lowercase_ , y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]   )   )
  return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,						_KWARGS_DESCRIPTION							)
class       _snake_case   (	datasets.Metric							):


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name not in [
       "boolq",
       "cb",
       "copa",
       "multirc",
       "record",
       "rte",
       "wic",
       "wsc",
       "wsc.fixed",
       "axb",
       "axg",
   ]:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")
   return datasets.MetricInfo(
       description=_DESCRIPTION					,  citation=_CITATION					,  inputs_description=_KWARGS_DESCRIPTION					,  features=datasets.Features(self._get_feature_types())					,  codebase_urls=[]					,  reference_urls=[]					,  format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None					,  )


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name == "record":
     return {
         "predictions": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "prediction_text": datasets.Value("""string"""),
         },
         "references": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "answers": datasets.Sequence(datasets.Value("""string""")),
         },
     }
   elif self.config_name == "multirc":
     return {
         "predictions": {
             "idx": {
                 "answer": datasets.Value("""int64"""),
                 "paragraph": datasets.Value("""int64"""),
                 "question": datasets.Value("""int64"""),
             },
             "prediction": datasets.Value("""int64"""),
         },
         "references": datasets.Value("""int64"""),
     }
   else:
     return {
         "predictions": datasets.Value("""int64"""),
         "references": datasets.Value("""int64"""),
     }



 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if self.config_name == "axb":
     return {"matthews_correlation": matthews_corrcoef(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   elif self.config_name == "cb":
     return acc_and_fa(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  fa_avg="""macro""")
   elif self.config_name == "record":
     lowercase__  :   Optional[Any]    		=      [
         {
             """qas""": [
                 {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                 for ref in references
             ]
         }
     ]
     lowercase__  :   str    		=      {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
     return evaluate_record(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)[0]
   elif self.config_name == "multirc":
     return evaluate_multirc(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
   elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
     return {"accuracy": simple_accuracy(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   else:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")






