
'''simple docstring'''





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_SCREAMING_SNAKE_CASE    :      str          =			re.compile(r'\b(a|an|the)\b', re.UNICODE)

_SCREAMING_SNAKE_CASE    :      Optional[Any]          =			None

def 				SCREAMING_SNAKE_CASE     (      ) ->       List[str]:

				"""simple docstring"""







				A__  :	Any         =							argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''       )
				parser.add_argument('''data_file'''							,						metavar='''data.json'''							,						help='''Input data JSON file.'''       )
				parser.add_argument('''pred_file'''							,						metavar='''pred.json'''							,						help='''Model predictions.'''       )
				parser.add_argument(
				    '''--out-file'''							,						'''-o'''							,						metavar='''eval.json'''							,						help='''Write accuracy metrics to file (default is stdout).'''       )
				parser.add_argument(
				    '''--na-prob-file'''							,						'''-n'''							,						metavar='''na_prob.json'''							,						help='''Model estimates of probability of no answer.'''       )
				parser.add_argument(
				    '''--na-prob-thresh'''							,						'''-t'''							,						type=__UpperCamelCase							,						default=1.0							,						help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''							,						)
				parser.add_argument(
				    '''--out-image-dir'''							,						'''-p'''							,						metavar='''out_images'''							,						default=__UpperCamelCase							,						help='''Save precision-recall curves to directory.'''       )
				parser.add_argument('''--verbose'''							,						'''-v'''							,						action='''store_true'''       )
				if len(sys.argv       ) == 1:
								parser.print_help()
								sys.exit(1       )
				return parser.parse_args()

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						Any       ) ->       int:

				"""simple docstring"""







				A__  :	List[Any]         =							{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																A__  :	Union[str, Any]         =							bool(qa['''answers''']['''text''']       )
				return qid_to_has_ans

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						Tuple       ) ->       List[str]:

				"""simple docstring"""







				def remove_articles(__UpperCamelCase :						Optional[int]       ):
								return ARTICLES_REGEX.sub(''' '''							,						__UpperCamelCase       )

				def white_space_fix(__UpperCamelCase :						List[str]       ):
								return " ".join(text.split()       )

				def remove_punc(__UpperCamelCase :						List[Any]       ):
								A__  :	Any         =							set(string.punctuation       )
								return "".join(ch for ch in text if ch not in exclude       )

				def lower(__UpperCamelCase :						List[Any]       ):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(__UpperCamelCase       )       )       )       )

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						Any       ) ->       str:

				"""simple docstring"""







				if not s:
								return []
				return normalize_answer(__UpperCamelCase       ).split()

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						List[str]							,						__UpperCamelCase :						Optional[int]       ) ->       Tuple:

				"""simple docstring"""







				return int(normalize_answer(__UpperCamelCase       ) == normalize_answer(__UpperCamelCase       )       )

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						List[Any]							,						__UpperCamelCase :						Optional[Any]       ) ->       Dict:

				"""simple docstring"""







				A__  :	Union[str, Any]         =							get_tokens(__UpperCamelCase       )
				A__  :	Optional[Any]         =							get_tokens(__UpperCamelCase       )
				A__  :	str         =							collections.Counter(__UpperCamelCase       ) & collections.Counter(__UpperCamelCase       )
				A__  :	Optional[int]         =							sum(common.values()       )
				if len(__UpperCamelCase       ) == 0 or len(__UpperCamelCase       ) == 0:
								# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
								return int(gold_toks == pred_toks       )
				if num_same == 0:
								return 0
				A__  :	Union[str, Any]         =							1.0 * num_same / len(__UpperCamelCase       )
				A__  :	List[Any]         =							1.0 * num_same / len(__UpperCamelCase       )
				A__  :	List[Any]         =							(2 * precision * recall) / (precision + recall)
				return fa

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						Any							,						__UpperCamelCase :						int       ) ->       Optional[Any]:

				"""simple docstring"""







				A__  :	Any         =							{}
				A__  :	Dict         =							{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																A__  :	Union[str, Any]         =							qa['''id''']
																A__  :	int         =							[t for t in qa['''answers''']['''text'''] if normalize_answer(__UpperCamelCase       )]
																if not gold_answers:
																				# For unanswerable questions, only correct answer is empty string
																				A__  :	Union[str, Any]         =							['''''']
																if qid not in preds:
																				print(F"Missing prediction for {qid}"       )
																				continue
																A__  :	Optional[int]         =							preds[qid]
																# Take max over all gold answers
																A__  :	List[str]         =							max(compute_exact(__UpperCamelCase							,						__UpperCamelCase       ) for a in gold_answers       )
																A__  :	Optional[int]         =							max(compute_fa(__UpperCamelCase							,						__UpperCamelCase       ) for a in gold_answers       )
				return exact_scores, fa_scores

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						Union[str, Any]							,						__UpperCamelCase :						List[Any]							,						__UpperCamelCase :						Tuple							,						__UpperCamelCase :						Any       ) ->       List[Any]:

				"""simple docstring"""







				A__  :	List[str]         =							{}
				for qid, s in scores.items():
								A__  :	Optional[Any]         =							na_probs[qid] > na_prob_thresh
								if pred_na:
												A__  :	Union[str, Any]         =							float(not qid_to_has_ans[qid]       )
								else:
												A__  :	Union[str, Any]         =							s
				return new_scores

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						str							,						__UpperCamelCase :						Optional[Any]							,						__UpperCamelCase :						Tuple=None       ) ->       Union[str, Any]:

				"""simple docstring"""







				if not qid_list:
								A__  :	Union[str, Any]         =							len(__UpperCamelCase       )
								return collections.OrderedDict(
								    [
								        ('''exact''', 1_00.0 * sum(exact_scores.values()       ) / total),
								        ('''f1''', 1_00.0 * sum(fa_scores.values()       ) / total),
								        ('''total''', total),
								    ]       )
				else:
								A__  :	Union[str, Any]         =							len(__UpperCamelCase       )
								return collections.OrderedDict(
								    [
								        ('''exact''', 1_00.0 * sum(exact_scores[k] for k in qid_list       ) / total),
								        ('''f1''', 1_00.0 * sum(fa_scores[k] for k in qid_list       ) / total),
								        ('''total''', total),
								    ]       )

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						List[str]							,						__UpperCamelCase :						Optional[int]							,						__UpperCamelCase :						int       ) ->       Optional[Any]:

				"""simple docstring"""







				for k in new_eval:
								A__  :	Optional[int]         =							new_eval[k]

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						List[Any]							,						__UpperCamelCase :						List[str]							,						__UpperCamelCase :						Optional[int]							,						__UpperCamelCase :						Dict       ) ->       Union[str, Any]:

				"""simple docstring"""







				plt.step(__UpperCamelCase							,						__UpperCamelCase							,						color='''b'''							,						alpha=0.2							,						where='''post'''       )
				plt.fill_between(__UpperCamelCase							,						__UpperCamelCase							,						step='''post'''							,						alpha=0.2							,						color='''b'''       )
				plt.xlabel('''Recall'''       )
				plt.ylabel('''Precision'''       )
				plt.xlim([0.0, 1.0_5]       )
				plt.ylim([0.0, 1.0_5]       )
				plt.title(__UpperCamelCase       )
				plt.savefig(__UpperCamelCase       )
				plt.clf()

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						List[str]							,						__UpperCamelCase :						Tuple							,						__UpperCamelCase :						Optional[int]							,						__UpperCamelCase :						int							,						__UpperCamelCase :						Optional[int]=None							,						__UpperCamelCase :						Optional[Any]=None       ) ->       Optional[Any]:

				"""simple docstring"""







				A__  :	int         =							sorted(__UpperCamelCase							,						key=lambda __UpperCamelCase       : na_probs[k]       )
				A__  :	int         =							0.0
				A__  :	int         =							1.0
				A__  :	Optional[int]         =							0.0
				A__  :	Tuple         =							[1.0]
				A__  :	Union[str, Any]         =							[0.0]
				A__  :	Any         =							0.0
				for i, qid in enumerate(__UpperCamelCase       ):
								if qid_to_has_ans[qid]:
												true_pos += scores[qid]
								A__  :	List[str]         =							true_pos / float(i + 1       )
								A__  :	List[str]         =							true_pos / float(__UpperCamelCase       )
								if i == len(__UpperCamelCase       ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
												# i.e., if we can put a threshold after this point
												avg_prec += cur_p * (cur_r - recalls[-1])
												precisions.append(__UpperCamelCase       )
												recalls.append(__UpperCamelCase       )
				if out_image:
								plot_pr_curve(__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase       )
				return {"ap": 1_00.0 * avg_prec}

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						Dict							,						__UpperCamelCase :						List[str]							,						__UpperCamelCase :						Optional[int]							,						__UpperCamelCase :						Dict							,						__UpperCamelCase :						List[str]							,						__UpperCamelCase :						List[Any]       ) ->       List[str]:

				"""simple docstring"""







				if out_image_dir and not os.path.exists(__UpperCamelCase       ):
								os.makedirs(__UpperCamelCase       )
				A__  :	Optional[Any]         =							sum(1 for v in qid_to_has_ans.values() if v       )
				if num_true_pos == 0:
								return
				A__  :	List[str]         =							make_precision_recall_eval(
				    __UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						out_image=os.path.join(__UpperCamelCase							,						'''pr_exact.png'''       )							,						title='''Precision-Recall curve for Exact Match score'''							,						)
				A__  :	List[Any]         =							make_precision_recall_eval(
				    __UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						out_image=os.path.join(__UpperCamelCase							,						'''pr_f1.png'''       )							,						title='''Precision-Recall curve for F1 score'''							,						)
				A__  :	Optional[Any]         =							{k: float(__UpperCamelCase       ) for k, v in qid_to_has_ans.items()}
				A__  :	List[str]         =							make_precision_recall_eval(
				    __UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						out_image=os.path.join(__UpperCamelCase							,						'''pr_oracle.png'''       )							,						title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''							,						)
				merge_eval(__UpperCamelCase							,						__UpperCamelCase							,						'''pr_exact'''       )
				merge_eval(__UpperCamelCase							,						__UpperCamelCase							,						'''pr_f1'''       )
				merge_eval(__UpperCamelCase							,						__UpperCamelCase							,						'''pr_oracle'''       )

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						List[str]							,						__UpperCamelCase :						List[Any]							,						__UpperCamelCase :						List[Any]							,						__UpperCamelCase :						Optional[Any]       ) ->       Optional[int]:

				"""simple docstring"""







				if not qid_list:
								return
				A__  :	str         =							[na_probs[k] for k in qid_list]
				A__  :	int         =							np.ones_like(__UpperCamelCase       ) / float(len(__UpperCamelCase       )       )
				plt.hist(__UpperCamelCase							,						weights=__UpperCamelCase							,						bins=20							,						range=(0.0, 1.0)       )
				plt.xlabel('''Model probability of no-answer'''       )
				plt.ylabel('''Proportion of dataset'''       )
				plt.title(F"Histogram of no-answer probability: {name}"       )
				plt.savefig(os.path.join(__UpperCamelCase							,						F"na_prob_hist_{name}.png"       )       )
				plt.clf()

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						str							,						__UpperCamelCase :						int							,						__UpperCamelCase :						Tuple							,						__UpperCamelCase :						List[Any]       ) ->       Any:

				"""simple docstring"""







				A__  :	List[str]         =							sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]       )
				A__  :	Any         =							num_no_ans
				A__  :	Union[str, Any]         =							cur_score
				A__  :	Tuple         =							0.0
				A__  :	int         =							sorted(__UpperCamelCase							,						key=lambda __UpperCamelCase       : na_probs[k]       )
				for i, qid in enumerate(__UpperCamelCase       ):
								if qid not in scores:
												continue
								if qid_to_has_ans[qid]:
												A__  :	Tuple         =							scores[qid]
								else:
												if preds[qid]:
																A__  :	Any         =							-1
												else:
																A__  :	Tuple         =							0
								cur_score += diff
								if cur_score > best_score:
												A__  :	Tuple         =							cur_score
												A__  :	Any         =							na_probs[qid]
				return 1_00.0 * best_score / len(__UpperCamelCase       ), best_thresh

def 				SCREAMING_SNAKE_CASE     (      __UpperCamelCase :						List[Any]							,						__UpperCamelCase :						int							,						__UpperCamelCase :						List[Any]							,						__UpperCamelCase :						int							,						__UpperCamelCase :						int							,						__UpperCamelCase :						Any       ) ->       Dict:

				"""simple docstring"""







				A__  :	Union[str, Any]         =							find_best_thresh(__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase       )
				A__  :	Union[str, Any]         =							find_best_thresh(__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase       )
				A__  :	Optional[Any]         =							best_exact
				A__  :	Optional[Any]         =							exact_thresh
				A__  :	str         =							best_fa
				A__  :	Tuple         =							fa_thresh

def 				SCREAMING_SNAKE_CASE     (      ) ->       List[Any]:

				"""simple docstring"""







				with open(OPTS.data_file       ) as f:
								A__  :	Tuple         =							json.load(__UpperCamelCase       )
								A__  :	List[Any]         =							dataset_json['''data''']
				with open(OPTS.pred_file       ) as f:
								A__  :	Optional[int]         =							json.load(__UpperCamelCase       )
				if OPTS.na_prob_file:
								with open(OPTS.na_prob_file       ) as f:
												A__  :	Union[str, Any]         =							json.load(__UpperCamelCase       )
				else:
								A__  :	Optional[int]         =							{k: 0.0 for k in preds}
				A__  :	List[str]         =							make_qid_to_has_ans(__UpperCamelCase       )  # maps qid to True/False
				A__  :	Optional[Any]         =							[k for k, v in qid_to_has_ans.items() if v]
				A__  :	List[Any]         =							[k for k, v in qid_to_has_ans.items() if not v]
				A__  :	int         =							get_raw_scores(__UpperCamelCase							,						__UpperCamelCase       )
				A__  :	Optional[Any]         =							apply_no_ans_threshold(__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						OPTS.na_prob_thresh       )
				A__  :	Dict         =							apply_no_ans_threshold(__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						OPTS.na_prob_thresh       )
				A__  :	List[Any]         =							make_eval_dict(__UpperCamelCase							,						__UpperCamelCase       )
				if has_ans_qids:
								A__  :	Optional[Any]         =							make_eval_dict(__UpperCamelCase							,						__UpperCamelCase							,						qid_list=__UpperCamelCase       )
								merge_eval(__UpperCamelCase							,						__UpperCamelCase							,						'''HasAns'''       )
				if no_ans_qids:
								A__  :	int         =							make_eval_dict(__UpperCamelCase							,						__UpperCamelCase							,						qid_list=__UpperCamelCase       )
								merge_eval(__UpperCamelCase							,						__UpperCamelCase							,						'''NoAns'''       )
				if OPTS.na_prob_file:
								find_all_best_thresh(__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase       )
				if OPTS.na_prob_file and OPTS.out_image_dir:
								run_precision_recall_analysis(__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						__UpperCamelCase							,						OPTS.out_image_dir       )
								histogram_na_prob(__UpperCamelCase							,						__UpperCamelCase							,						OPTS.out_image_dir							,						'''hasAns'''       )
								histogram_na_prob(__UpperCamelCase							,						__UpperCamelCase							,						OPTS.out_image_dir							,						'''noAns'''       )
				if OPTS.out_file:
								with open(OPTS.out_file							,						'''w'''       ) as f:
												json.dump(__UpperCamelCase							,						__UpperCamelCase       )
				else:
								print(json.dumps(__UpperCamelCase							,						indent=2       )       )


if __name__ == "__main__":
					_SCREAMING_SNAKE_CASE    :      Union[str, Any]          =			parse_args()
					if OPTS.out_image_dir:
										import matplotlib

										matplotlib.use('Agg')
										import matplotlib.pyplot as plt
					main()

import sacrebleu as scb
from packaging import version
from sacrebleu import TER

import datasets


_SCREAMING_SNAKE_CASE    :      Union[str, Any]          =			'\\n@inproceedings{snover-etal-2006-study,\n    title = "A Study of Translation Edit Rate with Targeted Human Annotation",\n    author = "Snover, Matthew  and\n      Dorr, Bonnie  and\n      Schwartz, Rich  and\n      Micciulla, Linnea  and\n      Makhoul, John",\n    booktitle = "Proceedings of the 7th Conference of the Association for Machine Translation in the Americas: Technical Papers",\n    month = aug # " 8-12",\n    year = "2006",\n    address = "Cambridge, Massachusetts, USA",\n    publisher = "Association for Machine Translation in the Americas",\n    url = "https://aclanthology.org/2006.amta-papers.25",\n    pages = "223--231",\n}\n@inproceedings{post-2018-call,\n    title = "A Call for Clarity in Reporting {BLEU} Scores",\n    author = "Post, Matt",\n    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",\n    month = oct,\n    year = "2018",\n    address = "Belgium, Brussels",\n    publisher = "Association for Computational Linguistics",\n    url = "https://www.aclweb.org/anthology/W18-6319",\n    pages = "186--191",\n}\n'

_SCREAMING_SNAKE_CASE    :      Tuple          =			'\\nTER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a\nhypothesis requires to match a reference translation. We use the implementation that is already present in sacrebleu\n(https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the TERCOM implementation, which can be found\nhere: https://github.com/jhclark/tercom.\n\nThe implementation here is slightly different from sacrebleu in terms of the required input format. The length of\nthe references and hypotheses lists need to be the same, so you may need to transpose your references compared to\nsacrebleu\'s required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534\n\nSee the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information.\n'

_SCREAMING_SNAKE_CASE    :      Optional[Any]          =			'\nProduces TER scores alongside the number of edits and reference length.\n\nArgs:\n    predictions (list of str): The system stream (a sequence of segments).\n    references (list of list of str): A list of one or more reference streams (each a sequence of segments).\n    normalized (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`.\n    ignore_punct (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`.\n    support_zh_ja_chars (boolean): If `True`, tokenization/normalization supports processing of Chinese characters,\n                                    as well as Japanese Kanji, Hiragana, Katakana, and Phonetic Extensions of Katakana.\n                                    Only applies if `normalized = True`. Defaults to `False`.\n    case_sensitive (boolean): If `False`, makes all predictions and references lowercase to ignore differences in case. Defaults to `False`.\n\nReturns:\n    \'score\' (float): TER score (num_edits / sum_ref_lengths * 100)\n    \'num_edits\' (int): The cumulative number of edits\n    \'ref_length\' (float): The cumulative average reference length\n\nExamples:\n    Example 1:\n        >>> predictions = ["does this sentence match??",\n        ...                     "what about this sentence?",\n        ...                     "What did the TER metric user say to the developer?"]\n        >>> references = [["does this sentence match", "does this sentence match!?!"],\n        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"],\n        ...             ["Your jokes are...", "...TERrible"]]\n        >>> ter = datasets.load_metric("ter")\n        >>> results = ter.compute(predictions=predictions,\n        ...                         references=references,\n        ...                         case_sensitive=True)\n        >>> print(results)\n        {\'score\': 150.0, \'num_edits\': 15, \'ref_length\': 10.0}\n\n    Example 2:\n        >>> predictions = ["does this sentence match??",\n        ...                     "what about this sentence?"]\n        >>> references = [["does this sentence match", "does this sentence match!?!"],\n        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]]\n        >>> ter = datasets.load_metric("ter")\n        >>> results = ter.compute(predictions=predictions,\n        ...                         references=references,\n        ...                         case_sensitive=True)\n        >>> print(results)\n        {\'score\': 62.5, \'num_edits\': 5, \'ref_length\': 8.0}\n\n    Example 3:\n        >>> predictions = ["does this sentence match??",\n        ...                     "what about this sentence?"]\n        >>> references = [["does this sentence match", "does this sentence match!?!"],\n        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]]\n        >>> ter = datasets.load_metric("ter")\n        >>> results = ter.compute(predictions=predictions,\n        ...                         references=references,\n        ...                         normalized=True,\n        ...                         case_sensitive=True)\n        >>> print(results)\n        {\'score\': 57.14285714285714, \'num_edits\': 6, \'ref_length\': 10.5}\n\n    Example 4:\n        >>> predictions = ["does this sentence match??",\n        ...                     "what about this sentence?"]\n        >>> references = [["does this sentence match", "does this sentence match!?!"],\n        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]]\n        >>> ter = datasets.load_metric("ter")\n        >>> results = ter.compute(predictions=predictions,\n        ...                         references=references,\n        ...                         ignore_punct=True,\n        ...                         case_sensitive=False)\n        >>> print(results)\n        {\'score\': 0.0, \'num_edits\': 0, \'ref_length\': 8.0}\n\n    Example 5:\n        >>> predictions = ["does this sentence match??",\n        ...                    "what about this sentence?",\n        ...                    "What did the TER metric user say to the developer?"]\n        >>> references = [["does this sentence match", "does this sentence match!?!"],\n        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"],\n        ...             ["Your jokes are...", "...TERrible"]]\n        >>> ter = datasets.load_metric("ter")\n        >>> results = ter.compute(predictions=predictions,\n        ...                         references=references,\n        ...                         ignore_punct=True,\n        ...                         case_sensitive=False)\n        >>> print(results)\n        {\'score\': 100.0, \'num_edits\': 10, \'ref_length\': 10.0}\n'







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION,			_KWARGS_DESCRIPTION   )
class  UpperCamelCase__		(    datasets.Metric   ):







	'''simple docstring'''





	def   __snake_case				(			self		):
					if version.parse(scb.__version__		) < version.parse('''1.4.12'''		):
									raise ImportWarning(
									    '''To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn\'t match this condition.\n'''
									    '''You can install it with `pip install "sacrebleu>=1.4.12"`.'''		)
					return datasets.MetricInfo(
					    description=_DESCRIPTION				,       citation=_CITATION				,       homepage='''http://www.cs.umd.edu/~snover/tercom/'''				,       inputs_description=_KWARGS_DESCRIPTION				,       features=datasets.Features(
					        {
					            '''predictions''': datasets.Value('''string'''				,       id='''sequence'''		),
					            '''references''': datasets.Sequence(datasets.Value('''string'''				,       id='''sequence'''		)				,       id='''references'''		),
					        }		)				,       codebase_urls=['''https://github.com/mjpost/sacreBLEU#ter''']				,       reference_urls=[
					        '''https://github.com/jhclark/tercom''',
					    ]				,       )





	def   __snake_case				(			self				,       UpperCamelCase__				,       UpperCamelCase__				,       UpperCamelCase__ = False				,       UpperCamelCase__ = False				,       UpperCamelCase__ = False				,       UpperCamelCase__ = False				,       ):
					A__  :	List[Any]         =							len(references[0]		)
					if any(len(UpperCamelCase__		) != references_per_prediction for refs in references		):
									raise ValueError('''Sacrebleu requires the same number of references for each prediction'''		)
					A__  :	Dict         =							[[refs[i] for refs in references] for i in range(UpperCamelCase__		)]

					A__  :	Optional[Any]         =							TER(
					    normalized=UpperCamelCase__				,       no_punct=UpperCamelCase__				,       asian_support=UpperCamelCase__				,       case_sensitive=UpperCamelCase__				,       )
					A__  :	str         =							sb_ter.corpus_score(UpperCamelCase__				,       UpperCamelCase__		)

					return {"score": output.score, "num_edits": output.num_edits, "ref_length": output.ref_length}