





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCamelCase__   : Any					  = re.compile(r"""\b(a|an|the)\b""", re.UNICODE)

UpperCamelCase__   : Optional[Any]					  = None




def    A_(			):
	UpperCAmelCase_								=     argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""   )
	parser.add_argument("""data_file"""				,      metavar="""data.json"""				,      help="""Input data JSON file."""   )
	parser.add_argument("""pred_file"""				,      metavar="""pred.json"""				,      help="""Model predictions."""   )
	parser.add_argument(
	    """--out-file"""				,      """-o"""				,      metavar="""eval.json"""				,      help="""Write accuracy metrics to file (default is stdout)."""   )
	parser.add_argument(
	    """--na-prob-file"""				,      """-n"""				,      metavar="""na_prob.json"""				,      help="""Model estimates of probability of no answer."""   )
	parser.add_argument(
	    """--na-prob-thresh"""				,      """-t"""				,      type=A				,      default=1.0				,      help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""				,      )
	parser.add_argument(
	    """--out-image-dir"""				,      """-p"""				,      metavar="""out_images"""				,      default=A				,      help="""Save precision-recall curves to directory."""   )
	parser.add_argument("""--verbose"""				,      """-v"""				,      action="""store_true"""   )
	if len(sys.argv   ) == 1:
		parser.print_help()
		sys.exit(1   )
	return parser.parse_args()




def    A_(			A   ):
	UpperCAmelCase_								=     {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				UpperCAmelCase_								=     bool(qa["""answers"""]["""text"""]   )
	return qid_to_has_ans




def    A_(			A   ):

	def remove_articles(A   ):
		return ARTICLES_REGEX.sub(""" """				,      A   )

	def white_space_fix(A   ):
		return " ".join(text.split()   )

	def remove_punc(A   ):
		UpperCAmelCase_								=     set(string.punctuation   )
		return "".join(ch for ch in text if ch not in exclude   )

	def lower(A   ):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(A   )   )   )   )




def    A_(			A   ):
	if not s:
		return []
	return normalize_answer(A   ).split()




def    A_(			A				,      A   ):
	return int(normalize_answer(A   ) == normalize_answer(A   )   )




def    A_(			A				,      A   ):
	UpperCAmelCase_								=     get_tokens(A   )
	UpperCAmelCase_								=     get_tokens(A   )
	UpperCAmelCase_								=     collections.Counter(A   ) & collections.Counter(A   )
	UpperCAmelCase_								=     sum(common.values()   )
	if len(A   ) == 0 or len(A   ) == 0:
		# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
		return int(gold_toks == pred_toks   )
	if num_same == 0:
		return 0
	UpperCAmelCase_								=     1.0 * num_same / len(A   )
	UpperCAmelCase_								=     1.0 * num_same / len(A   )
	UpperCAmelCase_								=     (2 * precision * recall) / (precision + recall)
	return fa




def    A_(			A				,      A   ):
	UpperCAmelCase_								=     {}
	UpperCAmelCase_								=     {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				UpperCAmelCase_								=     qa["""id"""]
				UpperCAmelCase_								=     [t for t in qa["""answers"""]["""text"""] if normalize_answer(A   )]
				if not gold_answers:
					# For unanswerable questions, only correct answer is empty string
					UpperCAmelCase_								=     [""""""]
				if qid not in preds:
					print(f"""Missing prediction for {qid}"""   )
					continue
				UpperCAmelCase_								=     preds[qid]
				# Take max over all gold answers
				UpperCAmelCase_								=     max(compute_exact(A				,      A   ) for a in gold_answers   )
				UpperCAmelCase_								=     max(compute_fa(A				,      A   ) for a in gold_answers   )
	return exact_scores, fa_scores




def    A_(			A				,      A				,      A				,      A   ):
	UpperCAmelCase_								=     {}
	for qid, s in scores.items():
		UpperCAmelCase_								=     na_probs[qid] > na_prob_thresh
		if pred_na:
			UpperCAmelCase_								=     float(not qid_to_has_ans[qid]   )
		else:
			UpperCAmelCase_								=     s
	return new_scores




def    A_(			A				,      A				,      A=None   ):
	if not qid_list:
		UpperCAmelCase_								=     len(A   )
		return collections.OrderedDict(
		    [
		        ("""exact""", 100.0 * sum(exact_scores.values()   ) / total),
		        ("""f1""", 100.0 * sum(fa_scores.values()   ) / total),
		        ("""total""", total),
		    ]   )
	else:
		UpperCAmelCase_								=     len(A   )
		return collections.OrderedDict(
		    [
		        ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list   ) / total),
		        ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list   ) / total),
		        ("""total""", total),
		    ]   )




def    A_(			A				,      A				,      A   ):
	for k in new_eval:
		UpperCAmelCase_								=     new_eval[k]




def    A_(			A				,      A				,      A				,      A   ):
	plt.step(A				,      A				,      color="""b"""				,      alpha=0.2				,      where="""post"""   )
	plt.fill_between(A				,      A				,      step="""post"""				,      alpha=0.2				,      color="""b"""   )
	plt.xlabel("""Recall"""   )
	plt.ylabel("""Precision"""   )
	plt.xlim([0.0, 1.05]   )
	plt.ylim([0.0, 1.05]   )
	plt.title(A   )
	plt.savefig(A   )
	plt.clf()




def    A_(			A				,      A				,      A				,      A				,      A=None				,      A=None   ):
	UpperCAmelCase_								=     sorted(A				,      key=lambda A   : na_probs[k]   )
	UpperCAmelCase_								=     0.0
	UpperCAmelCase_								=     1.0
	UpperCAmelCase_								=     0.0
	UpperCAmelCase_								=     [1.0]
	UpperCAmelCase_								=     [0.0]
	UpperCAmelCase_								=     0.0
	for i, qid in enumerate(A   ):
		if qid_to_has_ans[qid]:
			true_pos += scores[qid]
		UpperCAmelCase_								=     true_pos / float(i + 1   )
		UpperCAmelCase_								=     true_pos / float(A   )
		if i == len(A   ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
			# i.e., if we can put a threshold after this point
			avg_prec += cur_p * (cur_r - recalls[-1])
			precisions.append(A   )
			recalls.append(A   )
	if out_image:
		plot_pr_curve(A				,      A				,      A				,      A   )
	return {"ap": 100.0 * avg_prec}




def    A_(			A				,      A				,      A				,      A				,      A				,      A   ):
	if out_image_dir and not os.path.exists(A   ):
		os.makedirs(A   )
	UpperCAmelCase_								=     sum(1 for v in qid_to_has_ans.values() if v   )
	if num_true_pos == 0:
		return
	UpperCAmelCase_								=     make_precision_recall_eval(
	    A				,      A				,      A				,      A				,      out_image=os.path.join(A				,      """pr_exact.png"""   )				,      title="""Precision-Recall curve for Exact Match score"""				,      )
	UpperCAmelCase_								=     make_precision_recall_eval(
	    A				,      A				,      A				,      A				,      out_image=os.path.join(A				,      """pr_f1.png"""   )				,      title="""Precision-Recall curve for F1 score"""				,      )
	UpperCAmelCase_								=     {k: float(A   ) for k, v in qid_to_has_ans.items()}
	UpperCAmelCase_								=     make_precision_recall_eval(
	    A				,      A				,      A				,      A				,      out_image=os.path.join(A				,      """pr_oracle.png"""   )				,      title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""				,      )
	merge_eval(A				,      A				,      """pr_exact"""   )
	merge_eval(A				,      A				,      """pr_f1"""   )
	merge_eval(A				,      A				,      """pr_oracle"""   )




def    A_(			A				,      A				,      A				,      A   ):
	if not qid_list:
		return
	UpperCAmelCase_								=     [na_probs[k] for k in qid_list]
	UpperCAmelCase_								=     np.ones_like(A   ) / float(len(A   )   )
	plt.hist(A				,      weights=A				,      bins=20				,      range=(0.0, 1.0)   )
	plt.xlabel("""Model probability of no-answer"""   )
	plt.ylabel("""Proportion of dataset"""   )
	plt.title(f"""Histogram of no-answer probability: {name}"""   )
	plt.savefig(os.path.join(A				,      f"""na_prob_hist_{name}.png"""   )   )
	plt.clf()




def    A_(			A				,      A				,      A				,      A   ):
	UpperCAmelCase_								=     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]   )
	UpperCAmelCase_								=     num_no_ans
	UpperCAmelCase_								=     cur_score
	UpperCAmelCase_								=     0.0
	UpperCAmelCase_								=     sorted(A				,      key=lambda A   : na_probs[k]   )
	for i, qid in enumerate(A   ):
		if qid not in scores:
			continue
		if qid_to_has_ans[qid]:
			UpperCAmelCase_								=     scores[qid]
		else:
			if preds[qid]:
				UpperCAmelCase_								=     -1
			else:
				UpperCAmelCase_								=     0
		cur_score += diff
		if cur_score > best_score:
			UpperCAmelCase_								=     cur_score
			UpperCAmelCase_								=     na_probs[qid]
	return 100.0 * best_score / len(A   ), best_thresh




def    A_(			A				,      A				,      A				,      A				,      A				,      A   ):
	UpperCAmelCase_						,	UpperCAmelCase_								=     find_best_thresh(A				,      A				,      A				,      A   )
	UpperCAmelCase_						,	UpperCAmelCase_								=     find_best_thresh(A				,      A				,      A				,      A   )
	UpperCAmelCase_								=     best_exact
	UpperCAmelCase_								=     exact_thresh
	UpperCAmelCase_								=     best_fa
	UpperCAmelCase_								=     fa_thresh




def    A_(			):
	with open(OPTS.data_file   ) as f:
		UpperCAmelCase_								=     json.load(A   )
		UpperCAmelCase_								=     dataset_json["""data"""]
	with open(OPTS.pred_file   ) as f:
		UpperCAmelCase_								=     json.load(A   )
	if OPTS.na_prob_file:
		with open(OPTS.na_prob_file   ) as f:
			UpperCAmelCase_								=     json.load(A   )
	else:
		UpperCAmelCase_								=     {k: 0.0 for k in preds}
	UpperCAmelCase_								=     make_qid_to_has_ans(A   )  # maps qid to True/False
	UpperCAmelCase_								=     [k for k, v in qid_to_has_ans.items() if v]
	UpperCAmelCase_								=     [k for k, v in qid_to_has_ans.items() if not v]
	UpperCAmelCase_						,	UpperCAmelCase_								=     get_raw_scores(A				,      A   )
	UpperCAmelCase_								=     apply_no_ans_threshold(A				,      A				,      A				,      OPTS.na_prob_thresh   )
	UpperCAmelCase_								=     apply_no_ans_threshold(A				,      A				,      A				,      OPTS.na_prob_thresh   )
	UpperCAmelCase_								=     make_eval_dict(A				,      A   )
	if has_ans_qids:
		UpperCAmelCase_								=     make_eval_dict(A				,      A				,      qid_list=A   )
		merge_eval(A				,      A				,      """HasAns"""   )
	if no_ans_qids:
		UpperCAmelCase_								=     make_eval_dict(A				,      A				,      qid_list=A   )
		merge_eval(A				,      A				,      """NoAns"""   )
	if OPTS.na_prob_file:
		find_all_best_thresh(A				,      A				,      A				,      A				,      A				,      A   )
	if OPTS.na_prob_file and OPTS.out_image_dir:
		run_precision_recall_analysis(A				,      A				,      A				,      A				,      A				,      OPTS.out_image_dir   )
		histogram_na_prob(A				,      A				,      OPTS.out_image_dir				,      """hasAns"""   )
		histogram_na_prob(A				,      A				,      OPTS.out_image_dir				,      """noAns"""   )
	if OPTS.out_file:
		with open(OPTS.out_file				,      """w"""   ) as f:
			json.dump(A				,      A   )
	else:
		print(json.dumps(A				,      indent=2   )   )


if __name__ == "__main__":
				UpperCamelCase__   : int					  = parse_args()
				if OPTS.out_image_dir:
								import matplotlib

								matplotlib.use("""Agg""")
								import matplotlib.pyplot as plt
				main()









from torch import nn







class   _UpperCamelCase  (       nn.Module  ):







					'''simple docstring'''

					def __init__(						self    :  Dict ,					__lowercase    :  List[str] ,					__lowercase    :  Dict		):






						'''simple docstring'''







						super().__init__()
						UpperCAmelCase_								=     class_size
						UpperCAmelCase_								=     embed_size
						# self.mlp1 = nn.Linear(embed_size, embed_size)
						# self.mlp2 = (nn.Linear(embed_size, class_size))
						UpperCAmelCase_								=     nn.Linear(__lowercase ,					__lowercase		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Any ,					__lowercase    :  Optional[Any]		):






						'''simple docstring'''







						UpperCAmelCase_								=     self.mlp(__lowercase		)
						return logits


