
'''simple docstring'''

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__snake_case:  List[Any]								=			re.compile(R"\b(a|an|the)\b", re.UNICODE)

__snake_case:  Union[str, Any]								=			None






def 	_snake_case						(      ):
		"""simple docstring"""







		a_      :	Optional[int]      =							argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""   )
		parser.add_argument("""data_file"""     ,		metavar="""data.json"""     ,		help="""Input data JSON file."""   )
		parser.add_argument("""pred_file"""     ,		metavar="""pred.json"""     ,		help="""Model predictions."""   )
		parser.add_argument(
		    """--out-file"""     ,		"""-o"""     ,		metavar="""eval.json"""     ,		help="""Write accuracy metrics to file (default is stdout)."""   )
		parser.add_argument(
		    """--na-prob-file"""     ,		"""-n"""     ,		metavar="""na_prob.json"""     ,		help="""Model estimates of probability of no answer."""   )
		parser.add_argument(
		    """--na-prob-thresh"""     ,		"""-t"""     ,		type=A_     ,		default=1.0     ,		help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""     ,		)
		parser.add_argument(
		    """--out-image-dir"""     ,		"""-p"""     ,		metavar="""out_images"""     ,		default=A_     ,		help="""Save precision-recall curves to directory."""   )
		parser.add_argument("""--verbose"""     ,		"""-v"""     ,		action="""store_true"""   )
		if len(sys.argv   ) == 1:
				parser.print_help()
				sys.exit(1   )
		return parser.parse_args()






def 	_snake_case						(      A_       :    Optional[Any]   ):
		"""simple docstring"""







		a_      :	List[str]      =							{}
		for article in dataset:
				for p in article["paragraphs"]:
						for qa in p["qas"]:
								a_      :	int      =							bool(qa["""answers"""]["""text"""]   )
		return qid_to_has_ans






def 	_snake_case						(      A_       :    List[str]   ):
		"""simple docstring"""







		def remove_articles(A_       :    int   ):
				return ARTICLES_REGEX.sub(""" """     ,		A_   )

		def white_space_fix(A_       :    int   ):
				return " ".join(text.split()   )

		def remove_punc(A_       :    List[str]   ):
				a_      :	Optional[Any]      =							set(string.punctuation   )
				return "".join(ch for ch in text if ch not in exclude   )

		def lower(A_       :    List[str]   ):
				return text.lower()

		return white_space_fix(remove_articles(remove_punc(lower(A_   )   )   )   )






def 	_snake_case						(      A_       :    Any   ):
		"""simple docstring"""







		if not s:
				return []
		return normalize_answer(A_   ).split()






def 	_snake_case						(      A_       :    str     ,		A_       :    Tuple   ):
		"""simple docstring"""







		return int(normalize_answer(A_   ) == normalize_answer(A_   )   )






def 	_snake_case						(      A_       :    Tuple     ,		A_       :    Optional[Any]   ):
		"""simple docstring"""







		a_      :	str      =							get_tokens(A_   )
		a_      :	Tuple      =							get_tokens(A_   )
		a_      :	List[str]      =							collections.Counter(A_   ) & collections.Counter(A_   )
		a_      :	Tuple      =							sum(common.values()   )
		if len(A_   ) == 0 or len(A_   ) == 0:
				# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
				return int(gold_toks == pred_toks   )
		if num_same == 0:
				return 0
		a_      :	Optional[int]      =							1.0 * num_same / len(A_   )
		a_      :	List[str]      =							1.0 * num_same / len(A_   )
		a_      :	List[Any]      =							(2 * precision * recall) / (precision + recall)
		return fa






def 	_snake_case						(      A_       :    Any     ,		A_       :    Any   ):
		"""simple docstring"""







		a_      :	List[str]      =							{}
		a_      :	str      =							{}
		for article in dataset:
				for p in article["paragraphs"]:
						for qa in p["qas"]:
								a_      :	Any      =							qa["""id"""]
								a_      :	Optional[Any]      =							[t for t in qa["""answers"""]["""text"""] if normalize_answer(A_   )]
								if not gold_answers:
										# For unanswerable questions, only correct answer is empty string
										a_      :	List[Any]      =							[""""""]
								if qid not in preds:
										print(f'''Missing prediction for {qid}'''   )
										continue
								a_      :	Any      =							preds[qid]
								# Take max over all gold answers
								a_      :	int      =							max(compute_exact(A_     ,		A_   ) for a in gold_answers   )
								a_      :	int      =							max(compute_fa(A_     ,		A_   ) for a in gold_answers   )
		return exact_scores, fa_scores






def 	_snake_case						(      A_       :    List[Any]     ,		A_       :    Union[str, Any]     ,		A_       :    int     ,		A_       :    List[str]   ):
		"""simple docstring"""







		a_      :	Union[str, Any]      =							{}
		for qid, s in scores.items():
				a_      :	Union[str, Any]      =							na_probs[qid] > na_prob_thresh
				if pred_na:
						a_      :	List[str]      =							float(not qid_to_has_ans[qid]   )
				else:
						a_      :	Union[str, Any]      =							s
		return new_scores






def 	_snake_case						(      A_       :    int     ,		A_       :    Union[str, Any]     ,		A_       :    Optional[int]=None   ):
		"""simple docstring"""







		if not qid_list:
				a_      :	Union[str, Any]      =							len(A_   )
				return collections.OrderedDict(
				    [
				        ("""exact""", 100.0 * sum(exact_scores.values()   ) / total),
				        ("""f1""", 100.0 * sum(fa_scores.values()   ) / total),
				        ("""total""", total),
				    ]   )
		else:
				a_      :	int      =							len(A_   )
				return collections.OrderedDict(
				    [
				        ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list   ) / total),
				        ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list   ) / total),
				        ("""total""", total),
				    ]   )






def 	_snake_case						(      A_       :    Union[str, Any]     ,		A_       :    Optional[Any]     ,		A_       :    int   ):
		"""simple docstring"""







		for k in new_eval:
				a_      :	Any      =							new_eval[k]






def 	_snake_case						(      A_       :    Optional[int]     ,		A_       :    Optional[int]     ,		A_       :    List[Any]     ,		A_       :    str   ):
		"""simple docstring"""







		plt.step(A_     ,		A_     ,		color="""b"""     ,		alpha=0.2     ,		where="""post"""   )
		plt.fill_between(A_     ,		A_     ,		step="""post"""     ,		alpha=0.2     ,		color="""b"""   )
		plt.xlabel("""Recall"""   )
		plt.ylabel("""Precision"""   )
		plt.xlim([0.0, 1.05]   )
		plt.ylim([0.0, 1.05]   )
		plt.title(A_   )
		plt.savefig(A_   )
		plt.clf()






def 	_snake_case						(      A_       :    List[Any]     ,		A_       :    Optional[Any]     ,		A_       :    Dict     ,		A_       :    List[Any]     ,		A_       :    Any=None     ,		A_       :    List[str]=None   ):
		"""simple docstring"""







		a_      :	Dict      =							sorted(A_     ,		key=lambda A_   : na_probs[k]   )
		a_      :	Optional[Any]      =							0.0
		a_      :	List[str]      =							1.0
		a_      :	List[Any]      =							0.0
		a_      :	Union[str, Any]      =							[1.0]
		a_      :	Dict      =							[0.0]
		a_      :	Tuple      =							0.0
		for i, qid in enumerate(A_   ):
				if qid_to_has_ans[qid]:
						true_pos += scores[qid]
				a_      :	int      =							true_pos / float(i + 1   )
				a_      :	Tuple      =							true_pos / float(A_   )
				if i == len(A_   ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
						# i.e., if we can put a threshold after this point
						avg_prec += cur_p * (cur_r - recalls[-1])
						precisions.append(A_   )
						recalls.append(A_   )
		if out_image:
				plot_pr_curve(A_     ,		A_     ,		A_     ,		A_   )
		return {"ap": 100.0 * avg_prec}






def 	_snake_case						(      A_       :    Optional[Any]     ,		A_       :    Optional[Any]     ,		A_       :    Any     ,		A_       :    Optional[Any]     ,		A_       :    List[str]     ,		A_       :    int   ):
		"""simple docstring"""







		if out_image_dir and not os.path.exists(A_   ):
				os.makedirs(A_   )
		a_      :	List[str]      =							sum(1 for v in qid_to_has_ans.values() if v   )
		if num_true_pos == 0:
				return
		a_      :	Union[str, Any]      =							make_precision_recall_eval(
		    A_     ,		A_     ,		A_     ,		A_     ,		out_image=os.path.join(A_     ,		"""pr_exact.png"""   )     ,		title="""Precision-Recall curve for Exact Match score"""     ,		)
		a_      :	List[str]      =							make_precision_recall_eval(
		    A_     ,		A_     ,		A_     ,		A_     ,		out_image=os.path.join(A_     ,		"""pr_f1.png"""   )     ,		title="""Precision-Recall curve for F1 score"""     ,		)
		a_      :	List[str]      =							{k: float(A_   ) for k, v in qid_to_has_ans.items()}
		a_      :	Tuple      =							make_precision_recall_eval(
		    A_     ,		A_     ,		A_     ,		A_     ,		out_image=os.path.join(A_     ,		"""pr_oracle.png"""   )     ,		title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""     ,		)
		merge_eval(A_     ,		A_     ,		"""pr_exact"""   )
		merge_eval(A_     ,		A_     ,		"""pr_f1"""   )
		merge_eval(A_     ,		A_     ,		"""pr_oracle"""   )






def 	_snake_case						(      A_       :    Optional[Any]     ,		A_       :    List[Any]     ,		A_       :    Dict     ,		A_       :    Union[str, Any]   ):
		"""simple docstring"""







		if not qid_list:
				return
		a_      :	Dict      =							[na_probs[k] for k in qid_list]
		a_      :	Any      =							np.ones_like(A_   ) / float(len(A_   )   )
		plt.hist(A_     ,		weights=A_     ,		bins=20     ,		range=(0.0, 1.0)   )
		plt.xlabel("""Model probability of no-answer"""   )
		plt.ylabel("""Proportion of dataset"""   )
		plt.title(f'''Histogram of no-answer probability: {name}'''   )
		plt.savefig(os.path.join(A_     ,		f'''na_prob_hist_{name}.png'''   )   )
		plt.clf()






def 	_snake_case						(      A_       :    List[Any]     ,		A_       :    Tuple     ,		A_       :    Union[str, Any]     ,		A_       :    Union[str, Any]   ):
		"""simple docstring"""







		a_      :	Optional[int]      =							sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]   )
		a_      :	List[Any]      =							num_no_ans
		a_      :	str      =							cur_score
		a_      :	List[Any]      =							0.0
		a_      :	List[Any]      =							sorted(A_     ,		key=lambda A_   : na_probs[k]   )
		for i, qid in enumerate(A_   ):
				if qid not in scores:
						continue
				if qid_to_has_ans[qid]:
						a_      :	Tuple      =							scores[qid]
				else:
						if preds[qid]:
								a_      :	Union[str, Any]      =							-1
						else:
								a_      :	str      =							0
				cur_score += diff
				if cur_score > best_score:
						a_      :	Union[str, Any]      =							cur_score
						a_      :	List[Any]      =							na_probs[qid]
		return 100.0 * best_score / len(A_   ), best_thresh






def 	_snake_case						(      A_       :    str     ,		A_       :    List[str]     ,		A_       :    Dict     ,		A_       :    List[str]     ,		A_       :    Dict     ,		A_       :    Optional[int]   ):
		"""simple docstring"""







		a_      :	str      =							find_best_thresh(A_     ,		A_     ,		A_     ,		A_   )
		a_      :	str      =							find_best_thresh(A_     ,		A_     ,		A_     ,		A_   )
		a_      :	Tuple      =							best_exact
		a_      :	List[str]      =							exact_thresh
		a_      :	str      =							best_fa
		a_      :	str      =							fa_thresh






def 	_snake_case						(      ):
		"""simple docstring"""







		with open(OPTS.data_file   ) as f:
				a_      :	str      =							json.load(A_   )
				a_      :	int      =							dataset_json["""data"""]
		with open(OPTS.pred_file   ) as f:
				a_      :	Optional[int]      =							json.load(A_   )
		if OPTS.na_prob_file:
				with open(OPTS.na_prob_file   ) as f:
						a_      :	List[str]      =							json.load(A_   )
		else:
				a_      :	Optional[int]      =							{k: 0.0 for k in preds}
		a_      :	Optional[int]      =							make_qid_to_has_ans(A_   )  # maps qid to True/False
		a_      :	Optional[Any]      =							[k for k, v in qid_to_has_ans.items() if v]
		a_      :	Dict      =							[k for k, v in qid_to_has_ans.items() if not v]
		a_      :	Any      =							get_raw_scores(A_     ,		A_   )
		a_      :	Any      =							apply_no_ans_threshold(A_     ,		A_     ,		A_     ,		OPTS.na_prob_thresh   )
		a_      :	Union[str, Any]      =							apply_no_ans_threshold(A_     ,		A_     ,		A_     ,		OPTS.na_prob_thresh   )
		a_      :	int      =							make_eval_dict(A_     ,		A_   )
		if has_ans_qids:
				a_      :	int      =							make_eval_dict(A_     ,		A_     ,		qid_list=A_   )
				merge_eval(A_     ,		A_     ,		"""HasAns"""   )
		if no_ans_qids:
				a_      :	str      =							make_eval_dict(A_     ,		A_     ,		qid_list=A_   )
				merge_eval(A_     ,		A_     ,		"""NoAns"""   )
		if OPTS.na_prob_file:
				find_all_best_thresh(A_     ,		A_     ,		A_     ,		A_     ,		A_     ,		A_   )
		if OPTS.na_prob_file and OPTS.out_image_dir:
				run_precision_recall_analysis(A_     ,		A_     ,		A_     ,		A_     ,		A_     ,		OPTS.out_image_dir   )
				histogram_na_prob(A_     ,		A_     ,		OPTS.out_image_dir     ,		"""hasAns"""   )
				histogram_na_prob(A_     ,		A_     ,		OPTS.out_image_dir     ,		"""noAns"""   )
		if OPTS.out_file:
				with open(OPTS.out_file     ,		"""w"""   ) as f:
						json.dump(A_     ,		A_   )
		else:
				print(json.dumps(A_     ,		indent=2   )   )


if __name__ == "__main__":
				__snake_case:  List[Any]								=			parse_args()
				if OPTS.out_image_dir:
								import matplotlib

								matplotlib.use("Agg")
								import matplotlib.pyplot as plt
				main()




'''simple docstring'''

import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case:  Optional[Any]								=			logging.get_logger(__name__)

__snake_case:  Tuple								=			{"vocab_file": "sentencepiece.model"}

__snake_case:  int								=			{
    "vocab_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/sentencepiece.model",
    },
}

__snake_case:  Union[str, Any]								=			{
    "google/rembert": 2_56,
}


class  _UpperCAmelCase ( lowerCAmelCase__    ):

     """simple docstring"""

     a_    	=					VOCAB_FILES_NAMES
     a_    	=					PRETRAINED_VOCAB_FILES_MAP
     a_    	=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


     def __init__(      self							,		lowerCAmelCase_							,		lowerCAmelCase_=False							,		lowerCAmelCase_=True							,		lowerCAmelCase_=True							,		lowerCAmelCase_="[CLS]"							,		lowerCAmelCase_="[SEP]"							,		lowerCAmelCase_="[UNK]"							,		lowerCAmelCase_="[SEP]"							,		lowerCAmelCase_="[PAD]"							,		lowerCAmelCase_="[CLS]"							,		lowerCAmelCase_="[MASK]"							,		**lowerCAmelCase_							,		):

       '''simple docstring'''

       super().__init__(
           do_lower_case=lowerCAmelCase_							,		remove_space=lowerCAmelCase_							,		keep_accents=lowerCAmelCase_							,		bos_token=lowerCAmelCase_							,		eos_token=lowerCAmelCase_							,		unk_token=lowerCAmelCase_							,		sep_token=lowerCAmelCase_							,		pad_token=lowerCAmelCase_							,		cls_token=lowerCAmelCase_							,		mask_token=lowerCAmelCase_							,		**lowerCAmelCase_							,		)

       a_      :	int      =							do_lower_case
       a_      :	int      =							remove_space
       a_      :	Dict      =							keep_accents
       a_      :	Union[str, Any]      =							vocab_file

       a_      :	List[str]      =							spm.SentencePieceProcessor()
       self.sp_model.Load(lowerCAmelCase_ )


     @property
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       return len(self.sp_model )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Dict      =							{self.convert_ids_to_tokens(lowerCAmelCase_ ): i for i in range(self.vocab_size )}
       vocab.update(self.added_tokens_encoder )
       return vocab


     def __getstate__(      self ):

       '''simple docstring'''

       a_      :	Union[str, Any]      =							self.__dict__.copy()
       a_      :	Optional[Any]      =							None
       return state


     def __setstate__(      self							,		lowerCAmelCase_ ):

       '''simple docstring'''

       a_      :	int      =							d
       a_      :	Optional[int]      =							spm.SentencePieceProcessor()
       self.sp_model.Load(self.vocab_file )


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_							,		lowerCAmelCase_=False ):

       '''simple docstring'''

       a_      :	Tuple      =							self.sp_model.EncodeAsPieces(lowerCAmelCase_ )
       return pieces


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_ ):

       '''simple docstring'''

       return self.sp_model.PieceToId(lowerCAmelCase_ )


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_ ):

       '''simple docstring'''

       return self.sp_model.IdToPiece(lowerCAmelCase_ )


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_ ):

       '''simple docstring'''

       a_      :	Tuple      =							self.sp_model.decode_pieces(lowerCAmelCase_ )
       return out_string


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_							,		lowerCAmelCase_ = None ):

       '''simple docstring'''

       a_      :	List[Any]      =							[self.sep_token_id]
       a_      :	Tuple      =							[self.cls_token_id]
       if token_ids_a is None:
         return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_							,		lowerCAmelCase_ = None							,		lowerCAmelCase_ = False ):

       '''simple docstring'''

       if already_has_special_tokens:
         if token_ids_a is not None:
           raise ValueError(
               """You should not supply a second sequence if the provided sequence of """
               """ids is already formatted with special tokens for the model.""" )
         return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
         return [1] + ([0] * len(lowerCAmelCase_ )) + [1] + ([0] * len(lowerCAmelCase_ )) + [1]
       return [1] + ([0] * len(lowerCAmelCase_ )) + [1]


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_							,		lowerCAmelCase_ = None ):

       '''simple docstring'''

       a_      :	Optional[int]      =							[self.sep_token_id]
       a_      :	Optional[Any]      =							[self.cls_token_id]

       if token_ids_a is None:
         return len(cls + token_ids_a + sep ) * [0]
       return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_							,		lowerCAmelCase_ = None ):

       '''simple docstring'''

       if not os.path.isdir(lowerCAmelCase_ ):
         logger.error("""Vocabulary path ({}) should be a directory""".format(lowerCAmelCase_ ) )
         return
       a_      :	List[str]      =							os.path.join(
           lowerCAmelCase_							,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

       if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCAmelCase_ ):
         copyfile(self.vocab_file							,		lowerCAmelCase_ )

       return (out_vocab_file,)


