





'''simple docstring'''

import time
import warnings
from abc import ABC
from copy import deepcopy
from typing import Optional

import torch

from ..utils import add_start_docstrings, logging


__lowerCamelCase : List[str]											=			logging.get_logger(__name__)


__lowerCamelCase : List[Any]											=			r"\n    Args:\n        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):\n            Indices of input sequence tokens in the vocabulary.\n\n            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n            [`PreTrainedTokenizer.__call__`] for details.\n\n            [What are input IDs?](../glossary#input-ids)\n        scores (`torch.FloatTensor` of shape `(batch_size, config.vocab_size)`):\n            Prediction scores of a language modeling head. These can be scores for each vocabulary token before SoftMax\n            or scores for each vocabulary token after SoftMax.\n        kwargs (`Dict[str, Any]`, *optional*):\n            Additional stopping criteria specific kwargs.\n\n    Return:\n        `bool`. `False` indicates we should continue, `True` indicates we should stop.\n\n"

class       UpperCAmelCase (						_lowercase  ):




		@add_start_docstrings(A__				)
		def __call__(self      :	int							,			A__      :	torch.LongTensor							,			A__      :	torch.FloatTensor							,			**A__      :	int				)					-> bool:
						raise NotImplementedError("StoppingCriteria needs to be subclassed"				)




class       UpperCAmelCase (						_lowercase  ):




		def __init__(self      :	Dict							,			A__      :	int							,			A__      :	Optional[int] = None				)					-> str:
						lowercase         = max_length
						lowercase         = max_position_embeddings





		@add_start_docstrings(A__				)
		def __call__(self      :	Tuple							,			A__      :	torch.LongTensor							,			A__      :	torch.FloatTensor							,			**A__      :	Optional[int]				)					-> bool:
						lowercase         = input_ids.shape[-1]
						lowercase         = cur_len >= self.max_length
						if self.max_position_embeddings is not None and not is_done and cur_len >= self.max_position_embeddings:
										logger.warning_once(
										    "This is a friendly reminder - the current text generation call will exceed the model's predefined "
										    f'maximum length ({self.max_position_embeddings}). Depending on the model, you may observe '
										    "exceptions, performance degradation, or nothing at all."				)
						return is_done




class       UpperCAmelCase (						_lowercase  ):




		def __init__(self      :	int							,			A__      :	int							,			A__      :	int				)					-> Optional[Any]:
						warnings.warn(
						    "The class `MaxNewTokensCriteria` is deprecated. "
						    f'Please use `MaxLengthCriteria(max_length={start_length + max_new_tokens})` '
						    "with `max_length = start_length + max_new_tokens` instead."							,			A__							,			)
						lowercase         = start_length
						lowercase         = max_new_tokens
						lowercase         = start_length + max_new_tokens





		@add_start_docstrings(A__				)
		def __call__(self      :	str							,			A__      :	torch.LongTensor							,			A__      :	torch.FloatTensor							,			**A__      :	Optional[int]				)					-> bool:
						return input_ids.shape[-1] >= self.max_length




class       UpperCAmelCase (						_lowercase  ):




		def __init__(self      :	List[str]							,			A__      :	float							,			A__      :	Optional[float] = None				)					-> Union[str, Any]:
						lowercase         = max_time
						lowercase         = time.time() if initial_timestamp is None else initial_timestamp





		@add_start_docstrings(A__				)
		def __call__(self      :	List[Any]							,			A__      :	torch.LongTensor							,			A__      :	torch.FloatTensor							,			**A__      :	Dict				)					-> bool:
						return time.time() - self.initial_timestamp > self.max_time




class       UpperCAmelCase (						_lowercase  ):




		@add_start_docstrings(A__				)
		def __call__(self      :	str							,			A__      :	torch.LongTensor							,			A__      :	torch.FloatTensor							,			**A__      :	Tuple				)					-> bool:
						return any(criteria(A__							,			A__				) for criteria in self				)





		@property
		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> Optional[int]:
						for stopping_criterium in self:
										if isinstance(A__							,			A__				):
														return stopping_criterium.max_length
										elif isinstance(A__							,			A__				):
														return stopping_criterium.max_length
						return None







def 							UpperCAmelCase_    (	lowerCAmelCase_		,							lowerCAmelCase_							):

				"""simple docstring"""






				lowercase         = stopping_criteria.max_length
				lowercase         = deepcopy(lowerCAmelCase_							)
				if stopping_max_length is not None and stopping_max_length != max_length:
								warnings.warn("You set different `max_length` for stopping criteria and `max_length` parameter"		,							lowerCAmelCase_							)
				elif stopping_max_length is None:
								new_stopping_criteria.append(MaxLengthCriteria(max_length=lowerCAmelCase_							)							)
				return new_stopping_criteria











'''simple docstring'''

from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__lowerCamelCase : str											=			"\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

__lowerCamelCase : Optional[int]											=			"\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

__lowerCamelCase : int											=			"\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"







def 							UpperCAmelCase_    (	lowerCAmelCase_		,							lowerCAmelCase_							):

				"""simple docstring"""






				return float((preds == labels).mean()							)







def 							UpperCAmelCase_    (	lowerCAmelCase_		,							lowerCAmelCase_		,							lowerCAmelCase_="binary"							):

				"""simple docstring"""






				lowercase         = simple_accuracy(lowerCAmelCase_		,							lowerCAmelCase_							)
				lowercase         = float(fa_score(y_true=lowerCAmelCase_		,							y_pred=lowerCAmelCase_		,							average=lowerCAmelCase_							)							)
				return {
				    "accuracy": acc,
				    "f1": fa,
				}







def 							UpperCAmelCase_    (	lowerCAmelCase_		,							lowerCAmelCase_							):

				"""simple docstring"""






				lowercase         = {}
				for id_pred, label in zip(lowerCAmelCase_		,							lowerCAmelCase_							):
								lowercase         = f'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
								lowercase         = id_pred["prediction"]
								if question_id in question_map:
												question_map[question_id].append((pred, label)							)
								else:
												lowercase         = [(pred, label)]
				lowercase   ,							lowercase         = [], []
				for question, preds_labels in question_map.items():
								lowercase   ,							lowercase         = zip(*lowerCAmelCase_							)
								lowercase         = fa_score(y_true=lowerCAmelCase_		,							y_pred=lowerCAmelCase_		,							average="macro"							)
								fas.append(lowerCAmelCase_							)
								lowercase         = int(sum(pred == label for pred, label in preds_labels							) == len(lowerCAmelCase_							)							)
								ems.append(lowerCAmelCase_							)
				lowercase         = float(sum(lowerCAmelCase_							) / len(lowerCAmelCase_							)							)
				lowercase         = sum(lowerCAmelCase_							) / len(lowerCAmelCase_							)
				lowercase         = float(fa_score(y_true=lowerCAmelCase_		,							y_pred=[id_pred["prediction"] for id_pred in ids_preds]							)							)
				return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,       _KWARGS_DESCRIPTION  )
class       UpperCAmelCase (						datasets.Metric  ):




		def 				UpperCAmelCase__   (self      :	Any				)					-> str:
						if self.config_name not in [
						    "boolq",
						    "cb",
						    "copa",
						    "multirc",
						    "record",
						    "rte",
						    "wic",
						    "wsc",
						    "wsc.fixed",
						    "axb",
						    "axg",
						]:
										raise KeyError(
										    "You should supply a configuration name selected in "
										    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"				)
						return datasets.MetricInfo(
						    description=_DESCRIPTION							,			citation=_CITATION							,			inputs_description=_KWARGS_DESCRIPTION							,			features=datasets.Features(self._get_feature_types()				)							,			codebase_urls=[]							,			reference_urls=[]							,			format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None							,			)




		def 				UpperCAmelCase__   (self      :	int				)					-> Optional[int]:
						if self.config_name == "record":
										return {
										    "predictions": {
										        "idx": {
										            "passage": datasets.Value("int64"				),
										            "query": datasets.Value("int64"				),
										        },
										        "prediction_text": datasets.Value("string"				),
										    },
										    "references": {
										        "idx": {
										            "passage": datasets.Value("int64"				),
										            "query": datasets.Value("int64"				),
										        },
										        "answers": datasets.Sequence(datasets.Value("string"				)				),
										    },
										}
						elif self.config_name == "multirc":
										return {
										    "predictions": {
										        "idx": {
										            "answer": datasets.Value("int64"				),
										            "paragraph": datasets.Value("int64"				),
										            "question": datasets.Value("int64"				),
										        },
										        "prediction": datasets.Value("int64"				),
										    },
										    "references": datasets.Value("int64"				),
										}
						else:
										return {
										    "predictions": datasets.Value("int64"				),
										    "references": datasets.Value("int64"				),
										}





		def 				UpperCAmelCase__   (self      :	Any							,			A__      :	int							,			A__      :	List[str]				)					-> Dict:
						if self.config_name == "axb":
										return {"matthews_correlation": matthews_corrcoef(A__							,			A__				)}
						elif self.config_name == "cb":
										return acc_and_fa(A__							,			A__							,			fa_avg="macro"				)
						elif self.config_name == "record":
										lowercase         = [
										    {
										        "qas": [
										            {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
										            for ref in references
										        ]
										    }
										]
										lowercase         = {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
										return evaluate_record(A__							,			A__				)[0]
						elif self.config_name == "multirc":
										return evaluate_multirc(A__							,			A__				)
						elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
										return {"accuracy": simple_accuracy(A__							,			A__				)}
						else:
										raise KeyError(
										    "You should supply a configuration name selected in "
										    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"				)




