



'''simple docstring'''



import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCAmelCase_     :       Optional[int]								=						re.compile(R'\b(a|an|the)\b', re.UNICODE)

lowerCAmelCase_     :       List[str]								=						None
def 					UpperCAmelCase   (						):
       SCREAMING_SNAKE_CASE					:      int     					=			argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''				)
       parser.add_argument('''data_file'''					,		metavar='''data.json'''					,		help='''Input data JSON file.'''				)
       parser.add_argument('''pred_file'''					,		metavar='''pred.json'''					,		help='''Model predictions.'''				)
       parser.add_argument(
           '''--out-file'''					,		'''-o'''					,		metavar='''eval.json'''					,		help='''Write accuracy metrics to file (default is stdout).'''				)
       parser.add_argument(
           '''--na-prob-file'''					,		'''-n'''					,		metavar='''na_prob.json'''					,		help='''Model estimates of probability of no answer.'''				)
       parser.add_argument(
           '''--na-prob-thresh'''					,		'''-t'''					,		type=__lowerCAmelCase					,		default=1.0					,		help='''Predict \"\" if no-answer probability exceeds this (default = 1.0).'''					,		)
       parser.add_argument(
           '''--out-image-dir'''					,		'''-p'''					,		metavar='''out_images'''					,		default=__lowerCAmelCase					,		help='''Save precision-recall curves to directory.'''				)
       parser.add_argument('''--verbose'''					,		'''-v'''					,		action='''store_true'''				)
       if len(sys.argv				) == 1:
              parser.print_help()
              sys.exit(1				)
       return parser.parse_args()
def 					UpperCAmelCase   (						A  :				Tuple				):
       SCREAMING_SNAKE_CASE					:      Optional[int]     					=			{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            SCREAMING_SNAKE_CASE					:      Optional[int]     					=			bool(qa['''answers''']['''text''']				)
       return qid_to_has_ans
def 					UpperCAmelCase   (						A  :				List[str]				):

       def remove_articles(A  :				List[Any]				):
              return ARTICLES_REGEX.sub(''' '''					,		__lowerCAmelCase				)

       def white_space_fix(A  :				List[Any]				):
              return " ".join(text.split()				)

       def remove_punc(A  :				Any				):
              SCREAMING_SNAKE_CASE					:      Tuple     					=			set(string.punctuation				)
              return "".join(ch for ch in text if ch not in exclude				)

       def lower(A  :				int				):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(__lowerCAmelCase				)				)				)				)
def 					UpperCAmelCase   (						A  :				List[Any]				):
       if not s:
              return []
       return normalize_answer(__lowerCAmelCase				).split()
def 					UpperCAmelCase   (						A  :				List[str]					,		A  :				Union[str, Any]				):
       return int(normalize_answer(__lowerCAmelCase				) == normalize_answer(__lowerCAmelCase				)				)
def 					UpperCAmelCase   (						A  :				Union[str, Any]					,		A  :				Dict				):
       SCREAMING_SNAKE_CASE					:      List[Any]     					=			get_tokens(__lowerCAmelCase				)
       SCREAMING_SNAKE_CASE					:      List[Any]     					=			get_tokens(__lowerCAmelCase				)
       SCREAMING_SNAKE_CASE					:      List[Any]     					=			collections.Counter(__lowerCAmelCase				) & collections.Counter(__lowerCAmelCase				)
       SCREAMING_SNAKE_CASE					:      List[str]     					=			sum(common.values()				)
       if len(__lowerCAmelCase				) == 0 or len(__lowerCAmelCase				) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks				)
       if num_same == 0:
              return 0
       SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			1.0 * num_same / len(__lowerCAmelCase				)
       SCREAMING_SNAKE_CASE					:      Tuple     					=			1.0 * num_same / len(__lowerCAmelCase				)
       SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			(2 * precision * recall) / (precision + recall)
       return fa
def 					UpperCAmelCase   (						A  :				List[str]					,		A  :				Optional[Any]				):
       SCREAMING_SNAKE_CASE					:      Any     					=			{}
       SCREAMING_SNAKE_CASE					:      List[Any]     					=			{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			qa["""id"""]
                            SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			[t for t in qa["""answers"""]["""text"""] if normalize_answer(__lowerCAmelCase				)]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   SCREAMING_SNAKE_CASE					:      List[Any]     					=			[""""""]
                            if qid not in preds:
                                   print(F"""Missing prediction for {qid}"""				)
                                   continue
                            SCREAMING_SNAKE_CASE					:      List[Any]     					=			preds[qid]
                            # Take max over all gold answers
                            SCREAMING_SNAKE_CASE					:      Any     					=			max(compute_exact(__lowerCAmelCase					,		__lowerCAmelCase				) for a in gold_answers				)
                            SCREAMING_SNAKE_CASE					:      str     					=			max(compute_fa(__lowerCAmelCase					,		__lowerCAmelCase				) for a in gold_answers				)
       return exact_scores, fa_scores
def 					UpperCAmelCase   (						A  :				Tuple					,		A  :				Optional[int]					,		A  :				Optional[int]					,		A  :				List[Any]				):
       SCREAMING_SNAKE_CASE					:      str     					=			{}
       for qid, s in scores.items():
              SCREAMING_SNAKE_CASE					:      Any     					=			na_probs[qid] > na_prob_thresh
              if pred_na:
                     SCREAMING_SNAKE_CASE					:      List[str]     					=			float(not qid_to_has_ans[qid]				)
              else:
                     SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			s
       return new_scores
def 					UpperCAmelCase   (						A  :				Optional[Any]					,		A  :				List[Any]					,		A  :				Any=None				):
       if not qid_list:
              SCREAMING_SNAKE_CASE					:      int     					=			len(__lowerCAmelCase				)
              return collections.OrderedDict(
                  [
                      ('''exact''', 100.0 * sum(exact_scores.values()				) / total),
                      ('''f1''', 100.0 * sum(fa_scores.values()				) / total),
                      ('''total''', total),
                  ]				)
       else:
              SCREAMING_SNAKE_CASE					:      str     					=			len(__lowerCAmelCase				)
              return collections.OrderedDict(
                  [
                      ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list				) / total),
                      ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list				) / total),
                      ('''total''', total),
                  ]				)
def 					UpperCAmelCase   (						A  :				Optional[int]					,		A  :				int					,		A  :				Optional[Any]				):
       for k in new_eval:
              SCREAMING_SNAKE_CASE					:      Optional[int]     					=			new_eval[k]
def 					UpperCAmelCase   (						A  :				Optional[Any]					,		A  :				Any					,		A  :				str					,		A  :				str				):
       plt.step(__lowerCAmelCase					,		__lowerCAmelCase					,		color='''b'''					,		alpha=0.2					,		where='''post'''				)
       plt.fill_between(__lowerCAmelCase					,		__lowerCAmelCase					,		step='''post'''					,		alpha=0.2					,		color='''b'''				)
       plt.xlabel('''Recall'''				)
       plt.ylabel('''Precision'''				)
       plt.xlim([0.0, 1.05]				)
       plt.ylim([0.0, 1.05]				)
       plt.title(__lowerCAmelCase				)
       plt.savefig(__lowerCAmelCase				)
       plt.clf()
def 					UpperCAmelCase   (						A  :				Optional[int]					,		A  :				Tuple					,		A  :				List[str]					,		A  :				Dict					,		A  :				Tuple=None					,		A  :				Dict=None				):
       SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			sorted(__lowerCAmelCase					,		key=lambda A				: na_probs[k]				)
       SCREAMING_SNAKE_CASE					:      List[Any]     					=			0.0
       SCREAMING_SNAKE_CASE					:      List[Any]     					=			1.0
       SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			0.0
       SCREAMING_SNAKE_CASE					:      List[str]     					=			[1.0]
       SCREAMING_SNAKE_CASE					:      List[str]     					=			[0.0]
       SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			0.0
       for i, qid in enumerate(__lowerCAmelCase				):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			true_pos / float(i + 1				)
              SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			true_pos / float(__lowerCAmelCase				)
              if i == len(__lowerCAmelCase				) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(__lowerCAmelCase				)
                     recalls.append(__lowerCAmelCase				)
       if out_image:
              plot_pr_curve(__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase				)
       return {"ap": 100.0 * avg_prec}
def 					UpperCAmelCase   (						A  :				Dict					,		A  :				Dict					,		A  :				Dict					,		A  :				Any					,		A  :				Optional[int]					,		A  :				Optional[int]				):
       if out_image_dir and not os.path.exists(__lowerCAmelCase				):
              os.makedirs(__lowerCAmelCase				)
       SCREAMING_SNAKE_CASE					:      List[Any]     					=			sum(1 for v in qid_to_has_ans.values() if v				)
       if num_true_pos == 0:
              return
       SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			make_precision_recall_eval(
           __lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		out_image=os.path.join(__lowerCAmelCase					,		'''pr_exact.png'''				)					,		title='''Precision-Recall curve for Exact Match score'''					,		)
       SCREAMING_SNAKE_CASE					:      Tuple     					=			make_precision_recall_eval(
           __lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		out_image=os.path.join(__lowerCAmelCase					,		'''pr_f1.png'''				)					,		title='''Precision-Recall curve for F1 score'''					,		)
       SCREAMING_SNAKE_CASE					:      Any     					=			{k: float(__lowerCAmelCase				) for k, v in qid_to_has_ans.items()}
       SCREAMING_SNAKE_CASE					:      Tuple     					=			make_precision_recall_eval(
           __lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		out_image=os.path.join(__lowerCAmelCase					,		'''pr_oracle.png'''				)					,		title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''					,		)
       merge_eval(__lowerCAmelCase					,		__lowerCAmelCase					,		'''pr_exact'''				)
       merge_eval(__lowerCAmelCase					,		__lowerCAmelCase					,		'''pr_f1'''				)
       merge_eval(__lowerCAmelCase					,		__lowerCAmelCase					,		'''pr_oracle'''				)
def 					UpperCAmelCase   (						A  :				str					,		A  :				Optional[Any]					,		A  :				List[Any]					,		A  :				Any				):
       if not qid_list:
              return
       SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			[na_probs[k] for k in qid_list]
       SCREAMING_SNAKE_CASE					:      Any     					=			np.ones_like(__lowerCAmelCase				) / float(len(__lowerCAmelCase				)				)
       plt.hist(__lowerCAmelCase					,		weights=__lowerCAmelCase					,		bins=20					,		range=(0.0, 1.0)				)
       plt.xlabel('''Model probability of no-answer'''				)
       plt.ylabel('''Proportion of dataset'''				)
       plt.title(F"""Histogram of no-answer probability: {name}"""				)
       plt.savefig(os.path.join(__lowerCAmelCase					,		F"""na_prob_hist_{name}.png"""				)				)
       plt.clf()
def 					UpperCAmelCase   (						A  :				Union[str, Any]					,		A  :				List[str]					,		A  :				int					,		A  :				Union[str, Any]				):
       SCREAMING_SNAKE_CASE					:      Optional[int]     					=			sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]				)
       SCREAMING_SNAKE_CASE					:      Any     					=			num_no_ans
       SCREAMING_SNAKE_CASE					:      int     					=			cur_score
       SCREAMING_SNAKE_CASE					:      List[Any]     					=			0.0
       SCREAMING_SNAKE_CASE					:      Any     					=			sorted(__lowerCAmelCase					,		key=lambda A				: na_probs[k]				)
       for i, qid in enumerate(__lowerCAmelCase				):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     SCREAMING_SNAKE_CASE					:      Any     					=			scores[qid]
              else:
                     if preds[qid]:
                            SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			-1
                     else:
                            SCREAMING_SNAKE_CASE					:      Tuple     					=			0
              cur_score += diff
              if cur_score > best_score:
                     SCREAMING_SNAKE_CASE					:      List[str]     					=			cur_score
                     SCREAMING_SNAKE_CASE					:      Dict     					=			na_probs[qid]
       return 100.0 * best_score / len(__lowerCAmelCase				), best_thresh
def 					UpperCAmelCase   (						A  :				int					,		A  :				Dict					,		A  :				Optional[Any]					,		A  :				List[Any]					,		A  :				Dict					,		A  :				Any				):
       SCREAMING_SNAKE_CASE					:      Dict     					=			find_best_thresh(__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase				)
       SCREAMING_SNAKE_CASE					:      List[Any]     					=			find_best_thresh(__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase				)
       SCREAMING_SNAKE_CASE					:      str     					=			best_exact
       SCREAMING_SNAKE_CASE					:      Any     					=			exact_thresh
       SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			best_fa
       SCREAMING_SNAKE_CASE					:      List[str]     					=			fa_thresh
def 					UpperCAmelCase   (						):
       with open(OPTS.data_file				) as f:
              SCREAMING_SNAKE_CASE					:      int     					=			json.load(__lowerCAmelCase				)
              SCREAMING_SNAKE_CASE					:      str     					=			dataset_json["""data"""]
       with open(OPTS.pred_file				) as f:
              SCREAMING_SNAKE_CASE					:      Dict     					=			json.load(__lowerCAmelCase				)
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file				) as f:
                     SCREAMING_SNAKE_CASE					:      int     					=			json.load(__lowerCAmelCase				)
       else:
              SCREAMING_SNAKE_CASE					:      Tuple     					=			{k: 0.0 for k in preds}
       SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			make_qid_to_has_ans(__lowerCAmelCase				)  # maps qid to True/False
       SCREAMING_SNAKE_CASE					:      Any     					=			[k for k, v in qid_to_has_ans.items() if v]
       SCREAMING_SNAKE_CASE					:      int     					=			[k for k, v in qid_to_has_ans.items() if not v]
       SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			get_raw_scores(__lowerCAmelCase					,		__lowerCAmelCase				)
       SCREAMING_SNAKE_CASE					:      Dict     					=			apply_no_ans_threshold(__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		OPTS.na_prob_thresh				)
       SCREAMING_SNAKE_CASE					:      str     					=			apply_no_ans_threshold(__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		OPTS.na_prob_thresh				)
       SCREAMING_SNAKE_CASE					:      List[Any]     					=			make_eval_dict(__lowerCAmelCase					,		__lowerCAmelCase				)
       if has_ans_qids:
              SCREAMING_SNAKE_CASE					:      int     					=			make_eval_dict(__lowerCAmelCase					,		__lowerCAmelCase					,		qid_list=__lowerCAmelCase				)
              merge_eval(__lowerCAmelCase					,		__lowerCAmelCase					,		'''HasAns'''				)
       if no_ans_qids:
              SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			make_eval_dict(__lowerCAmelCase					,		__lowerCAmelCase					,		qid_list=__lowerCAmelCase				)
              merge_eval(__lowerCAmelCase					,		__lowerCAmelCase					,		'''NoAns'''				)
       if OPTS.na_prob_file:
              find_all_best_thresh(__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase				)
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		__lowerCAmelCase					,		OPTS.out_image_dir				)
              histogram_na_prob(__lowerCAmelCase					,		__lowerCAmelCase					,		OPTS.out_image_dir					,		'''hasAns'''				)
              histogram_na_prob(__lowerCAmelCase					,		__lowerCAmelCase					,		OPTS.out_image_dir					,		'''noAns'''				)
       if OPTS.out_file:
              with open(OPTS.out_file					,		'''w'''				) as f:
                     json.dump(__lowerCAmelCase					,		__lowerCAmelCase				)
       else:
              print(json.dumps(__lowerCAmelCase					,		indent=2				)				)


if __name__ == "__main__":
       lowerCAmelCase_     :       Dict								=						parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('Agg')
              import matplotlib.pyplot as plt
       main()









'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowerCAmelCase_     :       Tuple								=						{
    'configuration_tapas': ['TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP', 'TapasConfig'],
    'tokenization_tapas': ['TapasTokenizer'],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_     :       Dict								=						[
							    'TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'TapasForMaskedLM',
							    'TapasForQuestionAnswering',
							    'TapasForSequenceClassification',
							    'TapasModel',
							    'TapasPreTrainedModel',
							    'load_tf_weights_in_tapas',
							]
try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_     :       str								=						[
							    'TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'TFTapasForMaskedLM',
							    'TFTapasForQuestionAnswering',
							    'TFTapasForSequenceClassification',
							    'TFTapasModel',
							    'TFTapasPreTrainedModel',
							]


if TYPE_CHECKING:
							from .configuration_tapas import TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP, TapasConfig
							from .tokenization_tapas import TapasTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tapas import (
														    TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TapasForMaskedLM,
														    TapasForQuestionAnswering,
														    TapasForSequenceClassification,
														    TapasModel,
														    TapasPreTrainedModel,
														    load_tf_weights_in_tapas,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_tapas import (
														    TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFTapasForMaskedLM,
														    TFTapasForQuestionAnswering,
														    TFTapasForSequenceClassification,
														    TFTapasModel,
														    TFTapasPreTrainedModel,
														)


else:
							import sys

							lowerCAmelCase_     :       List[Any]								=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




