





'''simple docstring'''

from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__lowerCamelCase : str											=			"\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

__lowerCamelCase : Optional[int]											=			"\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

__lowerCamelCase : int											=			"\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"







def 							UpperCAmelCase_    (	lowerCAmelCase_		,							lowerCAmelCase_							):

				"""simple docstring"""






				return float((preds == labels).mean()							)







def 							UpperCAmelCase_    (	lowerCAmelCase_		,							lowerCAmelCase_		,							lowerCAmelCase_="binary"							):

				"""simple docstring"""






				lowercase         = simple_accuracy(lowerCAmelCase_		,							lowerCAmelCase_							)
				lowercase         = float(fa_score(y_true=lowerCAmelCase_		,							y_pred=lowerCAmelCase_		,							average=lowerCAmelCase_							)							)
				return {
				    "accuracy": acc,
				    "f1": fa,
				}







def 							UpperCAmelCase_    (	lowerCAmelCase_		,							lowerCAmelCase_							):

				"""simple docstring"""






				lowercase         = {}
				for id_pred, label in zip(lowerCAmelCase_		,							lowerCAmelCase_							):
								lowercase         = f'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
								lowercase         = id_pred["prediction"]
								if question_id in question_map:
												question_map[question_id].append((pred, label)							)
								else:
												lowercase         = [(pred, label)]
				lowercase   ,							lowercase         = [], []
				for question, preds_labels in question_map.items():
								lowercase   ,							lowercase         = zip(*lowerCAmelCase_							)
								lowercase         = fa_score(y_true=lowerCAmelCase_		,							y_pred=lowerCAmelCase_		,							average="macro"							)
								fas.append(lowerCAmelCase_							)
								lowercase         = int(sum(pred == label for pred, label in preds_labels							) == len(lowerCAmelCase_							)							)
								ems.append(lowerCAmelCase_							)
				lowercase         = float(sum(lowerCAmelCase_							) / len(lowerCAmelCase_							)							)
				lowercase         = sum(lowerCAmelCase_							) / len(lowerCAmelCase_							)
				lowercase         = float(fa_score(y_true=lowerCAmelCase_		,							y_pred=[id_pred["prediction"] for id_pred in ids_preds]							)							)
				return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,       _KWARGS_DESCRIPTION  )
class       UpperCAmelCase (						datasets.Metric  ):




		def 				UpperCAmelCase__   (self      :	Any				)					-> str:
						if self.config_name not in [
						    "boolq",
						    "cb",
						    "copa",
						    "multirc",
						    "record",
						    "rte",
						    "wic",
						    "wsc",
						    "wsc.fixed",
						    "axb",
						    "axg",
						]:
										raise KeyError(
										    "You should supply a configuration name selected in "
										    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"				)
						return datasets.MetricInfo(
						    description=_DESCRIPTION							,			citation=_CITATION							,			inputs_description=_KWARGS_DESCRIPTION							,			features=datasets.Features(self._get_feature_types()				)							,			codebase_urls=[]							,			reference_urls=[]							,			format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None							,			)




		def 				UpperCAmelCase__   (self      :	int				)					-> Optional[int]:
						if self.config_name == "record":
										return {
										    "predictions": {
										        "idx": {
										            "passage": datasets.Value("int64"				),
										            "query": datasets.Value("int64"				),
										        },
										        "prediction_text": datasets.Value("string"				),
										    },
										    "references": {
										        "idx": {
										            "passage": datasets.Value("int64"				),
										            "query": datasets.Value("int64"				),
										        },
										        "answers": datasets.Sequence(datasets.Value("string"				)				),
										    },
										}
						elif self.config_name == "multirc":
										return {
										    "predictions": {
										        "idx": {
										            "answer": datasets.Value("int64"				),
										            "paragraph": datasets.Value("int64"				),
										            "question": datasets.Value("int64"				),
										        },
										        "prediction": datasets.Value("int64"				),
										    },
										    "references": datasets.Value("int64"				),
										}
						else:
										return {
										    "predictions": datasets.Value("int64"				),
										    "references": datasets.Value("int64"				),
										}





		def 				UpperCAmelCase__   (self      :	Any							,			A__      :	int							,			A__      :	List[str]				)					-> Dict:
						if self.config_name == "axb":
										return {"matthews_correlation": matthews_corrcoef(A__							,			A__				)}
						elif self.config_name == "cb":
										return acc_and_fa(A__							,			A__							,			fa_avg="macro"				)
						elif self.config_name == "record":
										lowercase         = [
										    {
										        "qas": [
										            {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
										            for ref in references
										        ]
										    }
										]
										lowercase         = {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
										return evaluate_record(A__							,			A__				)[0]
						elif self.config_name == "multirc":
										return evaluate_multirc(A__							,			A__				)
						elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
										return {"accuracy": simple_accuracy(A__							,			A__				)}
						else:
										raise KeyError(
										    "You should supply a configuration name selected in "
										    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"				)











'''simple docstring'''

import inspect
import unittest

from transformers import RegNetConfig
from transformers.file_utils import cached_property, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch
			from torch import nn

			from transformers import RegNetForImageClassification, RegNetModel
			from transformers.models.regnet.modeling_regnet import REGNET_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
			from PIL import Image

			from transformers import AutoImageProcessor

class       UpperCAmelCase :




		def __init__(self      :	Dict							,			A__      :	str							,			A__      :	int=3							,			A__      :	Dict=3_2							,			A__      :	str=3							,			A__      :	str=1_0							,			A__      :	Optional[int]=[1_0, 2_0, 3_0, 4_0]							,			A__      :	Tuple=[1, 1, 2, 1]							,			A__      :	int=True							,			A__      :	List[Any]=True							,			A__      :	List[Any]="relu"							,			A__      :	Any=3							,			A__      :	Any=None							,			)					-> Tuple:
						lowercase         = parent
						lowercase         = batch_size
						lowercase         = image_size
						lowercase         = num_channels
						lowercase         = embeddings_size
						lowercase         = hidden_sizes
						lowercase         = depths
						lowercase         = is_training
						lowercase         = use_labels
						lowercase         = hidden_act
						lowercase         = num_labels
						lowercase         = scope
						lowercase         = len(A__				)




		def 				UpperCAmelCase__   (self      :	str				)					-> List[str]:
						lowercase         = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

						lowercase         = None
						if self.use_labels:
										lowercase         = ids_tensor([self.batch_size]							,			self.num_labels				)

						lowercase         = self.get_config()

						return config, pixel_values, labels




		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> List[str]:
						return RegNetConfig(
						    num_channels=self.num_channels							,			embeddings_size=self.embeddings_size							,			hidden_sizes=self.hidden_sizes							,			depths=self.depths							,			hidden_act=self.hidden_act							,			num_labels=self.num_labels							,			)




		def 				UpperCAmelCase__   (self      :	Dict							,			A__      :	Union[str, Any]							,			A__      :	Optional[int]							,			A__      :	Dict				)					-> str:
						lowercase         = RegNetModel(config=A__				)
						model.to(A__				)
						model.eval()
						lowercase         = model(A__				)
						# expected last hidden states: B, C, H // 32, W // 32
						self.parent.assertEqual(
						    result.last_hidden_state.shape							,			(self.batch_size, self.hidden_sizes[-1], self.image_size // 3_2, self.image_size // 3_2)							,			)




		def 				UpperCAmelCase__   (self      :	List[str]							,			A__      :	List[str]							,			A__      :	Union[str, Any]							,			A__      :	str				)					-> Dict:
						lowercase         = self.num_labels
						lowercase         = RegNetForImageClassification(A__				)
						model.to(A__				)
						model.eval()
						lowercase         = model(A__							,			labels=A__				)
						self.parent.assertEqual(result.logits.shape							,			(self.batch_size, self.num_labels)				)





		def 				UpperCAmelCase__   (self      :	Any				)					-> Union[str, Any]:
						lowercase         = self.prepare_config_and_inputs()
						lowercase   ,							lowercase   ,							lowercase         = config_and_inputs
						lowercase         = {"pixel_values": pixel_values}
						return config, inputs_dict




@require_torch
class       UpperCAmelCase (						_lowercase			,       _lowercase			,       unittest.TestCase  ):
		UpperCAmelCase					:	str 			=		(RegNetModel, RegNetForImageClassification) if is_torch_available() else ()
		UpperCAmelCase					:	Dict 			=		(
		    {'''feature-extraction''': RegNetModel, '''image-classification''': RegNetForImageClassification}
		    if is_torch_available()
		    else {}
		)

		UpperCAmelCase					:	Dict 			=		False
		UpperCAmelCase					:	int 			=		False
		UpperCAmelCase					:	Tuple 			=		False
		UpperCAmelCase					:	Tuple 			=		False




		def 				UpperCAmelCase__   (self      :	Optional[int]				)					-> Tuple:
						lowercase         = RegNetModelTester(self				)
						lowercase         = ConfigTester(self							,			config_class=A__							,			has_text_modality=A__				)




		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> str:
						self.create_and_test_config_common_properties()
						self.config_tester.create_and_test_config_to_json_string()
						self.config_tester.create_and_test_config_to_json_file()
						self.config_tester.create_and_test_config_from_and_save_pretrained()
						self.config_tester.create_and_test_config_with_num_labels()
						self.config_tester.check_config_can_be_init_without_params()
						self.config_tester.check_config_arguments_init()




		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> int:
						return




		@unittest.skip(reason="RegNet does not use inputs_embeds"				)
		def 				UpperCAmelCase__   (self      :	int				)					-> Optional[int]:
						pass




		@unittest.skip(reason="RegNet does not support input and output embeddings"				)
		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> Optional[Any]:
						pass




		def 				UpperCAmelCase__   (self      :	Any				)					-> str:
						lowercase   ,							lowercase         = self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
										lowercase         = model_class(A__				)
										lowercase         = inspect.signature(model.forward				)
										# signature.parameters is an OrderedDict => so arg_names order is deterministic
										lowercase         = [*signature.parameters.keys()]

										lowercase         = ["pixel_values"]
										self.assertListEqual(arg_names[:1]							,			A__				)




		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> Union[str, Any]:
						lowercase         = self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*A__				)




		def 				UpperCAmelCase__   (self      :	List[str]				)					-> Tuple:
						lowercase   ,							lowercase         = self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
										lowercase         = model_class(config=A__				)
										for name, module in model.named_modules():
														if isinstance(A__							,			(nn.BatchNormad, nn.GroupNorm)				):
																		self.assertTrue(
																		    torch.all(module.weight == 1				)							,			msg=f'Parameter {name} of model {model_class} seems not properly initialized'							,			)
																		self.assertTrue(
																		    torch.all(module.bias == 0				)							,			msg=f'Parameter {name} of model {model_class} seems not properly initialized'							,			)




		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> List[str]:
						def check_hidden_states_output(A__      :	Optional[Any]							,			A__      :	List[str]							,			A__      :	Tuple				):
										lowercase         = model_class(A__				)
										model.to(A__				)
										model.eval()

										with torch.no_grad():
														lowercase         = model(**self._prepare_for_class(A__							,			A__				)				)

										lowercase         = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

										lowercase         = self.model_tester.num_stages
										self.assertEqual(len(A__				)							,			expected_num_stages + 1				)

										# RegNet's feature maps are of shape (batch_size, num_channels, height, width)
										self.assertListEqual(
										    list(hidden_states[0].shape[-2:]				)							,			[self.model_tester.image_size // 2, self.model_tester.image_size // 2]							,			)

						lowercase   ,							lowercase         = self.model_tester.prepare_config_and_inputs_for_common()
						lowercase         = ["basic", "bottleneck"]
						for model_class in self.all_model_classes:
										for layer_type in layers_type:
														lowercase         = layer_type
														lowercase         = True
														check_hidden_states_output(A__							,			A__							,			A__				)

														# check that output_hidden_states also work using config
														del inputs_dict["output_hidden_states"]
														lowercase         = True

														check_hidden_states_output(A__							,			A__							,			A__				)




		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> Tuple:
						lowercase         = self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_image_classification(*A__				)





		@slow
		def 				UpperCAmelCase__   (self      :	Tuple				)					-> Any:
						for model_name in REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										lowercase         = RegNetModel.from_pretrained(A__				)
										self.assertIsNotNone(A__				)







def 							UpperCAmelCase_    (	):

				"""simple docstring"""






				lowercase         = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"							)
				return image


@require_torch
@require_vision
class       UpperCAmelCase (						unittest.TestCase  ):




		@cached_property
		def 				UpperCAmelCase__   (self      :	Optional[int]				)					-> Union[str, Any]:
						return (
						    AutoImageProcessor.from_pretrained(REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]				)
						    if is_vision_available()
						    else None
						)





		@slow
		def 				UpperCAmelCase__   (self      :	List[str]				)					-> int:
						lowercase         = RegNetForImageClassification.from_pretrained(REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]				).to(A__				)

						lowercase         = self.default_image_processor
						lowercase         = prepare_img()
						lowercase         = image_processor(images=A__							,			return_tensors="pt"				).to(A__				)

						# forward pass
						with torch.no_grad():
										lowercase         = model(**A__				)

						# verify the logits
						lowercase         = torch.Size((1, 1_0_0_0)				)
						self.assertEqual(outputs.logits.shape							,			A__				)

						lowercase         = torch.tensor([-0.4_1_8_0, -1.5_0_5_1, -3.4_8_3_6]				).to(A__				)

						self.assertTrue(torch.allclose(outputs.logits[0, :3]							,			A__							,			atol=1e-4				)				)




