



import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase_								=     re.compile(r"""\b(a|an|the)\b""", re.UNICODE)

lowercase_								=     None



def __lowerCAmelCase (     ):
			lowercase__					  =				argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."  )
			parser.add_argument("data_file"						,  metavar="data.json"						,  help="Input data JSON file."  )
			parser.add_argument("pred_file"						,  metavar="pred.json"						,  help="Model predictions."  )
			parser.add_argument(
			    "--out-file"						,  "-o"						,  metavar="eval.json"						,  help="Write accuracy metrics to file (default is stdout)."  )
			parser.add_argument(
			    "--na-prob-file"						,  "-n"						,  metavar="na_prob.json"						,  help="Model estimates of probability of no answer."  )
			parser.add_argument(
			    "--na-prob-thresh"						,  "-t"						,  type=SCREAMING_SNAKE_CASE_						,  default=1.0						,  help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."						,  )
			parser.add_argument(
			    "--out-image-dir"						,  "-p"						,  metavar="out_images"						,  default=SCREAMING_SNAKE_CASE_						,  help="Save precision-recall curves to directory."  )
			parser.add_argument("--verbose"						,  "-v"						,  action="store_true"  )
			if len(sys.argv  ) == 1:
						parser.print_help()
						sys.exit(1  )
			return parser.parse_args()



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				{}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												lowercase__					  =				bool(qa["answers"]["text"]  )
			return qid_to_has_ans



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):

			def remove_articles(SCREAMING_SNAKE_CASE_  ):
						return ARTICLES_REGEX.sub(" "						,  SCREAMING_SNAKE_CASE_  )

			def white_space_fix(SCREAMING_SNAKE_CASE_  ):
						return " ".join(text.split()  )

			def remove_punc(SCREAMING_SNAKE_CASE_  ):
						lowercase__					  =				set(string.punctuation  )
						return "".join(ch for ch in text if ch not in exclude  )

			def lower(SCREAMING_SNAKE_CASE_  ):
						return text.lower()

			return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE_  )  )  )  )



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			if not s:
						return []
			return normalize_answer(SCREAMING_SNAKE_CASE_  ).split()



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			return int(normalize_answer(SCREAMING_SNAKE_CASE_  ) == normalize_answer(SCREAMING_SNAKE_CASE_  )  )



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				get_tokens(SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				get_tokens(SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				collections.Counter(SCREAMING_SNAKE_CASE_  ) & collections.Counter(SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				sum(common.values()  )
			if len(SCREAMING_SNAKE_CASE_  ) == 0 or len(SCREAMING_SNAKE_CASE_  ) == 0:
						# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
						return int(gold_toks == pred_toks  )
			if num_same == 0:
						return 0
			lowercase__					  =				1.0 * num_same / len(SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				1.0 * num_same / len(SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				(2 * precision * recall) / (precision + recall)
			return fa



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				{}
			lowercase__					  =				{}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												lowercase__					  =				qa["id"]
												lowercase__					  =				[t for t in qa["answers"]["text"] if normalize_answer(SCREAMING_SNAKE_CASE_  )]
												if not gold_answers:
															# For unanswerable questions, only correct answer is empty string
															lowercase__					  =				[""]
												if qid not in preds:
															print(f'''Missing prediction for {qid}'''  )
															continue
												lowercase__					  =				preds[qid]
												# Take max over all gold answers
												lowercase__					  =				max(compute_exact(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ) for a in gold_answers  )
												lowercase__					  =				max(compute_fa(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ) for a in gold_answers  )
			return exact_scores, fa_scores



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				{}
			for qid, s in scores.items():
						lowercase__					  =				na_probs[qid] > na_prob_thresh
						if pred_na:
									lowercase__					  =				float(not qid_to_has_ans[qid]  )
						else:
									lowercase__					  =				s
			return new_scores



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_=None  ):
			if not qid_list:
						lowercase__					  =				len(SCREAMING_SNAKE_CASE_  )
						return collections.OrderedDict(
						    [
						        ("exact", 100.0 * sum(exact_scores.values()  ) / total),
						        ("f1", 100.0 * sum(fa_scores.values()  ) / total),
						        ("total", total),
						    ]  )
			else:
						lowercase__					  =				len(SCREAMING_SNAKE_CASE_  )
						return collections.OrderedDict(
						    [
						        ("exact", 100.0 * sum(exact_scores[k] for k in qid_list  ) / total),
						        ("f1", 100.0 * sum(fa_scores[k] for k in qid_list  ) / total),
						        ("total", total),
						    ]  )



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			for k in new_eval:
						lowercase__					  =				new_eval[k]



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			plt.step(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  color="b"						,  alpha=0.2						,  where="post"  )
			plt.fill_between(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  step="post"						,  alpha=0.2						,  color="b"  )
			plt.xlabel("Recall"  )
			plt.ylabel("Precision"  )
			plt.xlim([0.0, 1.05]  )
			plt.ylim([0.0, 1.05]  )
			plt.title(SCREAMING_SNAKE_CASE_  )
			plt.savefig(SCREAMING_SNAKE_CASE_  )
			plt.clf()



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_=None						,  SCREAMING_SNAKE_CASE_=None  ):
			lowercase__					  =				sorted(SCREAMING_SNAKE_CASE_						,  key=lambda SCREAMING_SNAKE_CASE_  : na_probs[k]  )
			lowercase__					  =				0.0
			lowercase__					  =				1.0
			lowercase__					  =				0.0
			lowercase__					  =				[1.0]
			lowercase__					  =				[0.0]
			lowercase__					  =				0.0
			for i, qid in enumerate(SCREAMING_SNAKE_CASE_  ):
						if qid_to_has_ans[qid]:
									true_pos += scores[qid]
						lowercase__					  =				true_pos / float(i + 1  )
						lowercase__					  =				true_pos / float(SCREAMING_SNAKE_CASE_  )
						if i == len(SCREAMING_SNAKE_CASE_  ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
									# i.e., if we can put a threshold after this point
									avg_prec += cur_p * (cur_r - recalls[-1])
									precisions.append(SCREAMING_SNAKE_CASE_  )
									recalls.append(SCREAMING_SNAKE_CASE_  )
			if out_image:
						plot_pr_curve(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			return {"ap": 100.0 * avg_prec}



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			if out_image_dir and not os.path.exists(SCREAMING_SNAKE_CASE_  ):
						os.makedirs(SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				sum(1 for v in qid_to_has_ans.values() if v  )
			if num_true_pos == 0:
						return
			lowercase__					  =				make_precision_recall_eval(
			    SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  out_image=os.path.join(SCREAMING_SNAKE_CASE_						,  "pr_exact.png"  )						,  title="Precision-Recall curve for Exact Match score"						,  )
			lowercase__					  =				make_precision_recall_eval(
			    SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  out_image=os.path.join(SCREAMING_SNAKE_CASE_						,  "pr_f1.png"  )						,  title="Precision-Recall curve for F1 score"						,  )
			lowercase__					  =				{k: float(SCREAMING_SNAKE_CASE_  ) for k, v in qid_to_has_ans.items()}
			lowercase__					  =				make_precision_recall_eval(
			    SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  out_image=os.path.join(SCREAMING_SNAKE_CASE_						,  "pr_oracle.png"  )						,  title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"						,  )
			merge_eval(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  "pr_exact"  )
			merge_eval(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  "pr_f1"  )
			merge_eval(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  "pr_oracle"  )



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			if not qid_list:
						return
			lowercase__					  =				[na_probs[k] for k in qid_list]
			lowercase__					  =				np.ones_like(SCREAMING_SNAKE_CASE_  ) / float(len(SCREAMING_SNAKE_CASE_  )  )
			plt.hist(SCREAMING_SNAKE_CASE_						,  weights=SCREAMING_SNAKE_CASE_						,  bins=20						,  range=(0.0, 1.0)  )
			plt.xlabel("Model probability of no-answer"  )
			plt.ylabel("Proportion of dataset"  )
			plt.title(f'''Histogram of no-answer probability: {name}'''  )
			plt.savefig(os.path.join(SCREAMING_SNAKE_CASE_						,  f'''na_prob_hist_{name}.png'''  )  )
			plt.clf()



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]  )
			lowercase__					  =				num_no_ans
			lowercase__					  =				cur_score
			lowercase__					  =				0.0
			lowercase__					  =				sorted(SCREAMING_SNAKE_CASE_						,  key=lambda SCREAMING_SNAKE_CASE_  : na_probs[k]  )
			for i, qid in enumerate(SCREAMING_SNAKE_CASE_  ):
						if qid not in scores:
									continue
						if qid_to_has_ans[qid]:
									lowercase__					  =				scores[qid]
						else:
									if preds[qid]:
												lowercase__					  =				-1
									else:
												lowercase__					  =				0
						cur_score += diff
						if cur_score > best_score:
									lowercase__					  =				cur_score
									lowercase__					  =				na_probs[qid]
			return 100.0 * best_score / len(SCREAMING_SNAKE_CASE_  ), best_thresh



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__  ,       lowercase__					  =				find_best_thresh(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			lowercase__  ,       lowercase__					  =				find_best_thresh(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				best_exact
			lowercase__					  =				exact_thresh
			lowercase__					  =				best_fa
			lowercase__					  =				fa_thresh



def __lowerCAmelCase (     ):
			with open(OPTS.data_file  ) as f:
						lowercase__					  =				json.load(SCREAMING_SNAKE_CASE_  )
						lowercase__					  =				dataset_json["data"]
			with open(OPTS.pred_file  ) as f:
						lowercase__					  =				json.load(SCREAMING_SNAKE_CASE_  )
			if OPTS.na_prob_file:
						with open(OPTS.na_prob_file  ) as f:
									lowercase__					  =				json.load(SCREAMING_SNAKE_CASE_  )
			else:
						lowercase__					  =				{k: 0.0 for k in preds}
			lowercase__					  =				make_qid_to_has_ans(SCREAMING_SNAKE_CASE_  )  # maps qid to True/False
			lowercase__					  =				[k for k, v in qid_to_has_ans.items() if v]
			lowercase__					  =				[k for k, v in qid_to_has_ans.items() if not v]
			lowercase__  ,       lowercase__					  =				get_raw_scores(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				apply_no_ans_threshold(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  OPTS.na_prob_thresh  )
			lowercase__					  =				apply_no_ans_threshold(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  OPTS.na_prob_thresh  )
			lowercase__					  =				make_eval_dict(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			if has_ans_qids:
						lowercase__					  =				make_eval_dict(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  qid_list=SCREAMING_SNAKE_CASE_  )
						merge_eval(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  "HasAns"  )
			if no_ans_qids:
						lowercase__					  =				make_eval_dict(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  qid_list=SCREAMING_SNAKE_CASE_  )
						merge_eval(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  "NoAns"  )
			if OPTS.na_prob_file:
						find_all_best_thresh(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			if OPTS.na_prob_file and OPTS.out_image_dir:
						run_precision_recall_analysis(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  OPTS.out_image_dir  )
						histogram_na_prob(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  OPTS.out_image_dir						,  "hasAns"  )
						histogram_na_prob(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  OPTS.out_image_dir						,  "noAns"  )
			if OPTS.out_file:
						with open(OPTS.out_file						,  "w"  ) as f:
									json.dump(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			else:
						print(json.dumps(SCREAMING_SNAKE_CASE_						,  indent=2  )  )


if __name__ == "__main__":
							lowercase_								=     parse_args()
							if OPTS.out_image_dir:
														import matplotlib

														matplotlib.use("""Agg""")
														import matplotlib.pyplot as plt
							main()









# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase_								=     {
    """configuration_xmod""": [
        """XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """XmodConfig""",
        """XmodOnnxConfig""",
    ],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase_								=     [
							    """XMOD_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """XmodForCausalLM""",
							    """XmodForMaskedLM""",
							    """XmodForMultipleChoice""",
							    """XmodForQuestionAnswering""",
							    """XmodForSequenceClassification""",
							    """XmodForTokenClassification""",
							    """XmodModel""",
							    """XmodPreTrainedModel""",
							]

if TYPE_CHECKING:
							from .configuration_xmod import XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP, XmodConfig, XmodOnnxConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_xmod import (
														    XMOD_PRETRAINED_MODEL_ARCHIVE_LIST,
														    XmodForCausalLM,
														    XmodForMaskedLM,
														    XmodForMultipleChoice,
														    XmodForQuestionAnswering,
														    XmodForSequenceClassification,
														    XmodForTokenClassification,
														    XmodModel,
														    XmodPreTrainedModel,
														)

else:
							import sys

							lowercase_								=     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




