



"""simple docstring"""

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


a_			   =					re.compile(r'''\b(a|an|the)\b''', re.UNICODE)

a_			   =					None







def 							SCREAMING_SNAKE_CASE__      (		):



					"""simple docstring"""





					snake_case_					:		int 					=					argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""			)
					parser.add_argument("""data_file"""				,    metavar="""data.json"""				,    help="""Input data JSON file."""			)
					parser.add_argument("""pred_file"""				,    metavar="""pred.json"""				,    help="""Model predictions."""			)
					parser.add_argument(
					    """--out-file"""				,    """-o"""				,    metavar="""eval.json"""				,    help="""Write accuracy metrics to file (default is stdout)."""			)
					parser.add_argument(
					    """--na-prob-file"""				,    """-n"""				,    metavar="""na_prob.json"""				,    help="""Model estimates of probability of no answer."""			)
					parser.add_argument(
					    """--na-prob-thresh"""				,    """-t"""				,    type=__snake_case				,    default=1.0				,    help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""				,    )
					parser.add_argument(
					    """--out-image-dir"""				,    """-p"""				,    metavar="""out_images"""				,    default=__snake_case				,    help="""Save precision-recall curves to directory."""			)
					parser.add_argument("""--verbose"""				,    """-v"""				,    action="""store_true"""			)
					if len(sys.argv			) == 1:
										parser.print_help()
										sys.exit(1			)
					return parser.parse_args()







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  int			):



					"""simple docstring"""





					snake_case_					:		Optional[int] 					=					{}
					for article in dataset:
										for p in article["paragraphs"]:
															for qa in p["qas"]:
																				snake_case_					:		Optional[Any] 					=					bool(qa["""answers"""]["""text"""]			)
					return qid_to_has_ans







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  str			):



					"""simple docstring"""





					def remove_articles(SCREAMING_SNAKE_CASE__ :  List[Any]			):
										return ARTICLES_REGEX.sub(""" """				,    __snake_case			)

					def white_space_fix(SCREAMING_SNAKE_CASE__ :  int			):
										return " ".join(text.split()			)

					def remove_punc(SCREAMING_SNAKE_CASE__ :  List[str]			):
										snake_case_					:		Union[str, Any] 					=					set(string.punctuation			)
										return "".join(ch for ch in text if ch not in exclude			)

					def lower(SCREAMING_SNAKE_CASE__ :  List[str]			):
										return text.lower()

					return white_space_fix(remove_articles(remove_punc(lower(__snake_case			)			)			)			)







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Optional[int]			):



					"""simple docstring"""





					if not s:
										return []
					return normalize_answer(__snake_case			).split()







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  str				,    SCREAMING_SNAKE_CASE__ :  Optional[Any]			):



					"""simple docstring"""





					return int(normalize_answer(__snake_case			) == normalize_answer(__snake_case			)			)







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  str				,    SCREAMING_SNAKE_CASE__ :  int			):



					"""simple docstring"""





					snake_case_					:		int 					=					get_tokens(__snake_case			)
					snake_case_					:		int 					=					get_tokens(__snake_case			)
					snake_case_					:		int 					=					collections.Counter(__snake_case			) & collections.Counter(__snake_case			)
					snake_case_					:		Optional[int] 					=					sum(common.values()			)
					if len(__snake_case			) == 0 or len(__snake_case			) == 0:
										# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
										return int(gold_toks == pred_toks			)
					if num_same == 0:
										return 0
					snake_case_					:		Union[str, Any] 					=					1.0 * num_same / len(__snake_case			)
					snake_case_					:		List[Any] 					=					1.0 * num_same / len(__snake_case			)
					snake_case_					:		List[Any] 					=					(2 * precision * recall) / (precision + recall)
					return fa







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Dict				,    SCREAMING_SNAKE_CASE__ :  Tuple			):



					"""simple docstring"""





					snake_case_					:		Optional[int] 					=					{}
					snake_case_					:		int 					=					{}
					for article in dataset:
										for p in article["paragraphs"]:
															for qa in p["qas"]:
																				snake_case_					:		str 					=					qa["id"]
																				snake_case_					:		Optional[int] 					=					[t for t in qa["answers"]["text"] if normalize_answer(__snake_case			)]
																				if not gold_answers:
																									# For unanswerable questions, only correct answer is empty string
																									snake_case_					:		Optional[int] 					=					[""]
																				if qid not in preds:
																									print(f'Missing prediction for {qid}'			)
																									continue
																				snake_case_					:		Optional[Any] 					=					preds[qid]
																				# Take max over all gold answers
																				snake_case_					:		Dict 					=					max(compute_exact(__snake_case				,    __snake_case			) for a in gold_answers			)
																				snake_case_					:		int 					=					max(compute_fa(__snake_case				,    __snake_case			) for a in gold_answers			)
					return exact_scores, fa_scores







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Optional[int]				,    SCREAMING_SNAKE_CASE__ :  List[str]				,    SCREAMING_SNAKE_CASE__ :  Union[str, Any]				,    SCREAMING_SNAKE_CASE__ :  Dict			):



					"""simple docstring"""





					snake_case_					:		Optional[Any] 					=					{}
					for qid, s in scores.items():
										snake_case_					:		Any 					=					na_probs[qid] > na_prob_thresh
										if pred_na:
															snake_case_					:		Optional[Any] 					=					float(not qid_to_has_ans[qid]			)
										else:
															snake_case_					:		Any 					=					s
					return new_scores







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Optional[int]				,    SCREAMING_SNAKE_CASE__ :  int				,    SCREAMING_SNAKE_CASE__ :  Union[str, Any]=None			):



					"""simple docstring"""





					if not qid_list:
										snake_case_					:		str 					=					len(__snake_case			)
										return collections.OrderedDict(
										    [
										        ("""exact""", 1_0_0.0 * sum(exact_scores.values()			) / total),
										        ("""f1""", 1_0_0.0 * sum(fa_scores.values()			) / total),
										        ("""total""", total),
										    ]			)
					else:
										snake_case_					:		Tuple 					=					len(__snake_case			)
										return collections.OrderedDict(
										    [
										        ("""exact""", 1_0_0.0 * sum(exact_scores[k] for k in qid_list			) / total),
										        ("""f1""", 1_0_0.0 * sum(fa_scores[k] for k in qid_list			) / total),
										        ("""total""", total),
										    ]			)







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  List[Any]				,    SCREAMING_SNAKE_CASE__ :  Dict				,    SCREAMING_SNAKE_CASE__ :  Dict			):



					"""simple docstring"""





					for k in new_eval:
										snake_case_					:		Tuple 					=					new_eval[k]







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Union[str, Any]				,    SCREAMING_SNAKE_CASE__ :  List[str]				,    SCREAMING_SNAKE_CASE__ :  Union[str, Any]				,    SCREAMING_SNAKE_CASE__ :  str			):



					"""simple docstring"""





					plt.step(__snake_case				,    __snake_case				,    color="""b"""				,    alpha=0.2				,    where="""post"""			)
					plt.fill_between(__snake_case				,    __snake_case				,    step="""post"""				,    alpha=0.2				,    color="""b"""			)
					plt.xlabel("""Recall"""			)
					plt.ylabel("""Precision"""			)
					plt.xlim([0.0, 1.05]			)
					plt.ylim([0.0, 1.05]			)
					plt.title(__snake_case			)
					plt.savefig(__snake_case			)
					plt.clf()







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Tuple				,    SCREAMING_SNAKE_CASE__ :  Dict				,    SCREAMING_SNAKE_CASE__ :  Optional[Any]				,    SCREAMING_SNAKE_CASE__ :  int				,    SCREAMING_SNAKE_CASE__ :  Optional[Any]=None				,    SCREAMING_SNAKE_CASE__ :  Any=None			):



					"""simple docstring"""





					snake_case_					:		Tuple 					=					sorted(__snake_case				,    key=lambda SCREAMING_SNAKE_CASE__			: na_probs[k]			)
					snake_case_					:		Tuple 					=					0.0
					snake_case_					:		List[Any] 					=					1.0
					snake_case_					:		Optional[Any] 					=					0.0
					snake_case_					:		Optional[Any] 					=					[1.0]
					snake_case_					:		str 					=					[0.0]
					snake_case_					:		int 					=					0.0
					for i, qid in enumerate(__snake_case			):
										if qid_to_has_ans[qid]:
															true_pos += scores[qid]
										snake_case_					:		int 					=					true_pos / float(i + 1			)
										snake_case_					:		int 					=					true_pos / float(__snake_case			)
										if i == len(__snake_case			) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
															# i.e., if we can put a threshold after this point
															avg_prec += cur_p * (cur_r - recalls[-1])
															precisions.append(__snake_case			)
															recalls.append(__snake_case			)
					if out_image:
										plot_pr_curve(__snake_case				,    __snake_case				,    __snake_case				,    __snake_case			)
					return {"ap": 1_0_0.0 * avg_prec}







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Union[str, Any]				,    SCREAMING_SNAKE_CASE__ :  int				,    SCREAMING_SNAKE_CASE__ :  str				,    SCREAMING_SNAKE_CASE__ :  Union[str, Any]				,    SCREAMING_SNAKE_CASE__ :  int				,    SCREAMING_SNAKE_CASE__ :  Tuple			):



					"""simple docstring"""





					if out_image_dir and not os.path.exists(__snake_case			):
										os.makedirs(__snake_case			)
					snake_case_					:		List[Any] 					=					sum(1 for v in qid_to_has_ans.values() if v			)
					if num_true_pos == 0:
										return
					snake_case_					:		Dict 					=					make_precision_recall_eval(
					    __snake_case				,    __snake_case				,    __snake_case				,    __snake_case				,    out_image=os.path.join(__snake_case				,    """pr_exact.png"""			)				,    title="""Precision-Recall curve for Exact Match score"""				,    )
					snake_case_					:		int 					=					make_precision_recall_eval(
					    __snake_case				,    __snake_case				,    __snake_case				,    __snake_case				,    out_image=os.path.join(__snake_case				,    """pr_f1.png"""			)				,    title="""Precision-Recall curve for F1 score"""				,    )
					snake_case_					:		Tuple 					=					{k: float(__snake_case			) for k, v in qid_to_has_ans.items()}
					snake_case_					:		Optional[int] 					=					make_precision_recall_eval(
					    __snake_case				,    __snake_case				,    __snake_case				,    __snake_case				,    out_image=os.path.join(__snake_case				,    """pr_oracle.png"""			)				,    title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""				,    )
					merge_eval(__snake_case				,    __snake_case				,    """pr_exact"""			)
					merge_eval(__snake_case				,    __snake_case				,    """pr_f1"""			)
					merge_eval(__snake_case				,    __snake_case				,    """pr_oracle"""			)







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Any				,    SCREAMING_SNAKE_CASE__ :  Any				,    SCREAMING_SNAKE_CASE__ :  Union[str, Any]				,    SCREAMING_SNAKE_CASE__ :  Any			):



					"""simple docstring"""





					if not qid_list:
										return
					snake_case_					:		Optional[Any] 					=					[na_probs[k] for k in qid_list]
					snake_case_					:		Optional[Any] 					=					np.ones_like(__snake_case			) / float(len(__snake_case			)			)
					plt.hist(__snake_case				,    weights=__snake_case				,    bins=2_0				,    range=(0.0, 1.0)			)
					plt.xlabel("""Model probability of no-answer"""			)
					plt.ylabel("""Proportion of dataset"""			)
					plt.title(f'Histogram of no-answer probability: {name}'			)
					plt.savefig(os.path.join(__snake_case				,    f'na_prob_hist_{name}.png'			)			)
					plt.clf()







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Optional[Any]				,    SCREAMING_SNAKE_CASE__ :  Optional[int]				,    SCREAMING_SNAKE_CASE__ :  List[Any]				,    SCREAMING_SNAKE_CASE__ :  List[Any]			):



					"""simple docstring"""





					snake_case_					:		int 					=					sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]			)
					snake_case_					:		str 					=					num_no_ans
					snake_case_					:		Any 					=					cur_score
					snake_case_					:		int 					=					0.0
					snake_case_					:		Any 					=					sorted(__snake_case				,    key=lambda SCREAMING_SNAKE_CASE__			: na_probs[k]			)
					for i, qid in enumerate(__snake_case			):
										if qid not in scores:
															continue
										if qid_to_has_ans[qid]:
															snake_case_					:		Any 					=					scores[qid]
										else:
															if preds[qid]:
																				snake_case_					:		Optional[int] 					=					-1
															else:
																				snake_case_					:		List[str] 					=					0
										cur_score += diff
										if cur_score > best_score:
															snake_case_					:		List[str] 					=					cur_score
															snake_case_					:		List[Any] 					=					na_probs[qid]
					return 1_0_0.0 * best_score / len(__snake_case			), best_thresh







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Optional[int]				,    SCREAMING_SNAKE_CASE__ :  Optional[Any]				,    SCREAMING_SNAKE_CASE__ :  List[str]				,    SCREAMING_SNAKE_CASE__ :  List[Any]				,    SCREAMING_SNAKE_CASE__ :  Union[str, Any]				,    SCREAMING_SNAKE_CASE__ :  List[str]			):



					"""simple docstring"""





					snake_case_					:		Tuple 					=					find_best_thresh(__snake_case				,    __snake_case				,    __snake_case				,    __snake_case			)
					snake_case_					:		Dict 					=					find_best_thresh(__snake_case				,    __snake_case				,    __snake_case				,    __snake_case			)
					snake_case_					:		Any 					=					best_exact
					snake_case_					:		Tuple 					=					exact_thresh
					snake_case_					:		Any 					=					best_fa
					snake_case_					:		str 					=					fa_thresh







def 							SCREAMING_SNAKE_CASE__      (		):



					"""simple docstring"""





					with open(OPTS.data_file			) as f:
										snake_case_					:		Any 					=					json.load(__snake_case			)
										snake_case_					:		Any 					=					dataset_json["data"]
					with open(OPTS.pred_file			) as f:
										snake_case_					:		List[str] 					=					json.load(__snake_case			)
					if OPTS.na_prob_file:
										with open(OPTS.na_prob_file			) as f:
															snake_case_					:		str 					=					json.load(__snake_case			)
					else:
										snake_case_					:		Optional[Any] 					=					{k: 0.0 for k in preds}
					snake_case_					:		List[str] 					=					make_qid_to_has_ans(__snake_case			)  # maps qid to True/False
					snake_case_					:		Tuple 					=					[k for k, v in qid_to_has_ans.items() if v]
					snake_case_					:		List[str] 					=					[k for k, v in qid_to_has_ans.items() if not v]
					snake_case_					:		Any 					=					get_raw_scores(__snake_case				,    __snake_case			)
					snake_case_					:		List[str] 					=					apply_no_ans_threshold(__snake_case				,    __snake_case				,    __snake_case				,    OPTS.na_prob_thresh			)
					snake_case_					:		List[str] 					=					apply_no_ans_threshold(__snake_case				,    __snake_case				,    __snake_case				,    OPTS.na_prob_thresh			)
					snake_case_					:		List[Any] 					=					make_eval_dict(__snake_case				,    __snake_case			)
					if has_ans_qids:
										snake_case_					:		Optional[Any] 					=					make_eval_dict(__snake_case				,    __snake_case				,    qid_list=__snake_case			)
										merge_eval(__snake_case				,    __snake_case				,    """HasAns"""			)
					if no_ans_qids:
										snake_case_					:		Any 					=					make_eval_dict(__snake_case				,    __snake_case				,    qid_list=__snake_case			)
										merge_eval(__snake_case				,    __snake_case				,    """NoAns"""			)
					if OPTS.na_prob_file:
										find_all_best_thresh(__snake_case				,    __snake_case				,    __snake_case				,    __snake_case				,    __snake_case				,    __snake_case			)
					if OPTS.na_prob_file and OPTS.out_image_dir:
										run_precision_recall_analysis(__snake_case				,    __snake_case				,    __snake_case				,    __snake_case				,    __snake_case				,    OPTS.out_image_dir			)
										histogram_na_prob(__snake_case				,    __snake_case				,    OPTS.out_image_dir				,    """hasAns"""			)
										histogram_na_prob(__snake_case				,    __snake_case				,    OPTS.out_image_dir				,    """noAns"""			)
					if OPTS.out_file:
										with open(OPTS.out_file				,    """w"""			) as f:
															json.dump(__snake_case				,    __snake_case			)
					else:
										print(json.dumps(__snake_case				,    indent=2			)			)


if __name__ == "__main__":
				a_			   =					parse_args()
				if OPTS.out_image_dir:
								import matplotlib

								matplotlib.use('''Agg''')
								import matplotlib.pyplot as plt
				main()









"""simple docstring"""

from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


if TYPE_CHECKING:
			from ... import FeatureExtractionMixin, PreTrainedTokenizerBase, TensorType


a_			   =					logging.get_logger(__name__)

a_			   =					{
    '''microsoft/deberta-v2-xlarge''': '''https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/config.json''',
    '''microsoft/deberta-v2-xxlarge''': '''https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/config.json''',
    '''microsoft/deberta-v2-xlarge-mnli''': (
        '''https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/config.json'''
    ),
    '''microsoft/deberta-v2-xxlarge-mnli''': (
        '''https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/config.json'''
    ),
}







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						_A		:							Dict							       =		"""deberta-v2"""







						def __init__(self   ,		lowercase__=12_81_00   ,		lowercase__=15_36   ,		lowercase__=24   ,		lowercase__=24   ,		lowercase__=61_44   ,		lowercase__="gelu"   ,		lowercase__=0.1   ,		lowercase__=0.1   ,		lowercase__=5_12   ,		lowercase__=0   ,		lowercase__=0.02   ,		lowercase__=1e-7   ,		lowercase__=False   ,		lowercase__=-1   ,		lowercase__=0   ,		lowercase__=True   ,		lowercase__=None   ,		lowercase__=0   ,		lowercase__="gelu"   ,		**lowercase__   ,		):
											super().__init__(**lowercase__       )

											snake_case_					:		Union[str, Any] 					=					hidden_size
											snake_case_					:		str 					=					num_hidden_layers
											snake_case_					:		Tuple 					=					num_attention_heads
											snake_case_					:		Dict 					=					intermediate_size
											snake_case_					:		Optional[int] 					=					hidden_act
											snake_case_					:		Union[str, Any] 					=					hidden_dropout_prob
											snake_case_					:		Any 					=					attention_probs_dropout_prob
											snake_case_					:		List[Any] 					=					max_position_embeddings
											snake_case_					:		Union[str, Any] 					=					type_vocab_size
											snake_case_					:		Union[str, Any] 					=					initializer_range
											snake_case_					:		List[Any] 					=					relative_attention
											snake_case_					:		Dict 					=					max_relative_positions
											snake_case_					:		Optional[int] 					=					pad_token_id
											snake_case_					:		List[str] 					=					position_biased_input

											# Backwards compatibility
											if type(lowercase__       ) == str:
																snake_case_					:		Union[str, Any] 					=					[x.strip() for x in pos_att_type.lower().split("""|"""       )]

											snake_case_					:		Optional[int] 					=					pos_att_type
											snake_case_					:		List[str] 					=					vocab_size
											snake_case_					:		Tuple 					=					layer_norm_eps

											snake_case_					:		List[Any] 					=					kwargs.get("""pooler_hidden_size"""   ,		lowercase__       )
											snake_case_					:		List[str] 					=					pooler_dropout
											snake_case_					:		int 					=					pooler_hidden_act







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						@property
						def 						__UpperCamelCase							(self       ):
											if self.task == "multiple-choice":
																snake_case_					:		List[Any] 					=					{0: """batch""", 1: """choice""", 2: """sequence"""}
											else:
																snake_case_					:		int 					=					{0: """batch""", 1: """sequence"""}
											if self._config.type_vocab_size > 0:
																return OrderedDict(
																    [("""input_ids""", dynamic_axis), ("""attention_mask""", dynamic_axis), ("""token_type_ids""", dynamic_axis)]       )
											else:
																return OrderedDict([("""input_ids""", dynamic_axis), ("""attention_mask""", dynamic_axis)]       )







						@property
						def 						__UpperCamelCase							(self       ):
											return 12







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__ = -1   ,		lowercase__ = -1   ,		lowercase__ = -1   ,		lowercase__ = False   ,		lowercase__ = None   ,		lowercase__ = 3   ,		lowercase__ = 40   ,		lowercase__ = 40   ,		lowercase__ = None   ,		):
											snake_case_					:		str 					=					super().generate_dummy_inputs(preprocessor=lowercase__   ,		framework=lowercase__       )
											if self._config.type_vocab_size == 0 and "token_type_ids" in dummy_inputs:
																del dummy_inputs["token_type_ids"]
											return dummy_inputs




