



import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__lowerCamelCase         =			re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

__lowerCamelCase         =			None







def    _snake_case (       )     ->     Optional[Any]:






      '''simple docstring'''







      UpperCAmelCase_  :	Optional[int]				      =		argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."    )
      parser.add_argument("data_file"					,  metavar="data.json"					,  help="Input data JSON file."    )
      parser.add_argument("pred_file"					,  metavar="pred.json"					,  help="Model predictions."    )
      parser.add_argument(
          "--out-file"					,  "-o"					,  metavar="eval.json"					,  help="Write accuracy metrics to file (default is stdout)."    )
      parser.add_argument(
          "--na-prob-file"					,  "-n"					,  metavar="na_prob.json"					,  help="Model estimates of probability of no answer."    )
      parser.add_argument(
          "--na-prob-thresh"					,  "-t"					,  type=__snake_case					,  default=1.0					,  help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."					,  )
      parser.add_argument(
          "--out-image-dir"					,  "-p"					,  metavar="out_images"					,  default=__snake_case					,  help="Save precision-recall curves to directory."    )
      parser.add_argument("--verbose"					,  "-v"					,  action="store_true"    )
      if len(sys.argv    ) == 1:
            parser.print_help()
            sys.exit(1    )
      return parser.parse_args()







def    _snake_case (       __snake_case    )     ->     List[str]:






      '''simple docstring'''







      UpperCAmelCase_  :	Tuple				      =		{}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        UpperCAmelCase_  :	Union[str, Any]				      =		bool(qa["answers"]["text"]    )
      return qid_to_has_ans







def    _snake_case (       __snake_case    )     ->     Union[str, Any]:






      '''simple docstring'''







      def remove_articles(__snake_case    ):
            return ARTICLES_REGEX.sub(" "					,  __snake_case    )

      def white_space_fix(__snake_case    ):
            return " ".join(text.split()    )

      def remove_punc(__snake_case    ):
            UpperCAmelCase_  :	str				      =		set(string.punctuation    )
            return "".join(ch for ch in text if ch not in exclude    )

      def lower(__snake_case    ):
            return text.lower()

      return white_space_fix(remove_articles(remove_punc(lower(__snake_case    )    )    )    )







def    _snake_case (       __snake_case    )     ->     Any:






      '''simple docstring'''







      if not s:
            return []
      return normalize_answer(__snake_case    ).split()







def    _snake_case (       __snake_case					,  __snake_case    )     ->     Optional[int]:






      '''simple docstring'''







      return int(normalize_answer(__snake_case    ) == normalize_answer(__snake_case    )    )







def    _snake_case (       __snake_case					,  __snake_case    )     ->     List[str]:






      '''simple docstring'''







      UpperCAmelCase_  :	str				      =		get_tokens(__snake_case    )
      UpperCAmelCase_  :	Union[str, Any]				      =		get_tokens(__snake_case    )
      UpperCAmelCase_  :	Optional[int]				      =		collections.Counter(__snake_case    ) & collections.Counter(__snake_case    )
      UpperCAmelCase_  :	List[str]				      =		sum(common.values()    )
      if len(__snake_case    ) == 0 or len(__snake_case    ) == 0:
            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
            return int(gold_toks == pred_toks    )
      if num_same == 0:
            return 0
      UpperCAmelCase_  :	str				      =		1.0 * num_same / len(__snake_case    )
      UpperCAmelCase_  :	Any				      =		1.0 * num_same / len(__snake_case    )
      UpperCAmelCase_  :	str				      =		(2 * precision * recall) / (precision + recall)
      return fa







def    _snake_case (       __snake_case					,  __snake_case    )     ->     List[str]:






      '''simple docstring'''







      UpperCAmelCase_  :	Union[str, Any]				      =		{}
      UpperCAmelCase_  :	Optional[Any]				      =		{}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        UpperCAmelCase_  :	str				      =		qa["id"]
                        UpperCAmelCase_  :	int				      =		[t for t in qa["answers"]["text"] if normalize_answer(__snake_case    )]
                        if not gold_answers:
                              # For unanswerable questions, only correct answer is empty string
                              UpperCAmelCase_  :	Tuple				      =		[""]
                        if qid not in preds:
                              print(F"""Missing prediction for {qid}"""    )
                              continue
                        UpperCAmelCase_  :	Optional[Any]				      =		preds[qid]
                        # Take max over all gold answers
                        UpperCAmelCase_  :	Union[str, Any]				      =		max(compute_exact(__snake_case					,  __snake_case    ) for a in gold_answers    )
                        UpperCAmelCase_  :	int				      =		max(compute_fa(__snake_case					,  __snake_case    ) for a in gold_answers    )
      return exact_scores, fa_scores







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )     ->     int:






      '''simple docstring'''







      UpperCAmelCase_  :	int				      =		{}
      for qid, s in scores.items():
            UpperCAmelCase_  :	Any				      =		na_probs[qid] > na_prob_thresh
            if pred_na:
                  UpperCAmelCase_  :	Any				      =		float(not qid_to_has_ans[qid]    )
            else:
                  UpperCAmelCase_  :	Tuple				      =		s
      return new_scores







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case=None    )     ->     Any:






      '''simple docstring'''







      if not qid_list:
            UpperCAmelCase_  :	Dict				      =		len(__snake_case    )
            return collections.OrderedDict(
                [
                    ("exact", 100.0 * sum(exact_scores.values()    ) / total),
                    ("f1", 100.0 * sum(fa_scores.values()    ) / total),
                    ("total", total),
                ]    )
      else:
            UpperCAmelCase_  :	str				      =		len(__snake_case    )
            return collections.OrderedDict(
                [
                    ("exact", 100.0 * sum(exact_scores[k] for k in qid_list    ) / total),
                    ("f1", 100.0 * sum(fa_scores[k] for k in qid_list    ) / total),
                    ("total", total),
                ]    )







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case    )     ->     Optional[Any]:






      '''simple docstring'''







      for k in new_eval:
            UpperCAmelCase_  :	List[str]				      =		new_eval[k]







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )     ->     Optional[int]:






      '''simple docstring'''







      plt.step(__snake_case					,  __snake_case					,  color="b"					,  alpha=0.2					,  where="post"    )
      plt.fill_between(__snake_case					,  __snake_case					,  step="post"					,  alpha=0.2					,  color="b"    )
      plt.xlabel("Recall"    )
      plt.ylabel("Precision"    )
      plt.xlim([0.0, 1.05]    )
      plt.ylim([0.0, 1.05]    )
      plt.title(__snake_case    )
      plt.savefig(__snake_case    )
      plt.clf()







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  __snake_case=None					,  __snake_case=None    )     ->     Tuple:






      '''simple docstring'''







      UpperCAmelCase_  :	Optional[Any]				      =		sorted(__snake_case					,  key=lambda __snake_case    : na_probs[k]    )
      UpperCAmelCase_  :	Dict				      =		0.0
      UpperCAmelCase_  :	List[str]				      =		1.0
      UpperCAmelCase_  :	Optional[Any]				      =		0.0
      UpperCAmelCase_  :	Tuple				      =		[1.0]
      UpperCAmelCase_  :	str				      =		[0.0]
      UpperCAmelCase_  :	Optional[Any]				      =		0.0
      for i, qid in enumerate(__snake_case    ):
            if qid_to_has_ans[qid]:
                  true_pos += scores[qid]
            UpperCAmelCase_  :	int				      =		true_pos / float(i + 1    )
            UpperCAmelCase_  :	Optional[Any]				      =		true_pos / float(__snake_case    )
            if i == len(__snake_case    ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                  # i.e., if we can put a threshold after this point
                  avg_prec += cur_p * (cur_r - recalls[-1])
                  precisions.append(__snake_case    )
                  recalls.append(__snake_case    )
      if out_image:
            plot_pr_curve(__snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )
      return {"ap": 100.0 * avg_prec}







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )     ->     str:






      '''simple docstring'''







      if out_image_dir and not os.path.exists(__snake_case    ):
            os.makedirs(__snake_case    )
      UpperCAmelCase_  :	Tuple				      =		sum(1 for v in qid_to_has_ans.values() if v    )
      if num_true_pos == 0:
            return
      UpperCAmelCase_  :	Optional[int]				      =		make_precision_recall_eval(
          __snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  out_image=os.path.join(__snake_case					,  "pr_exact.png"    )					,  title="Precision-Recall curve for Exact Match score"					,  )
      UpperCAmelCase_  :	Dict				      =		make_precision_recall_eval(
          __snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  out_image=os.path.join(__snake_case					,  "pr_f1.png"    )					,  title="Precision-Recall curve for F1 score"					,  )
      UpperCAmelCase_  :	List[Any]				      =		{k: float(__snake_case    ) for k, v in qid_to_has_ans.items()}
      UpperCAmelCase_  :	Dict				      =		make_precision_recall_eval(
          __snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  out_image=os.path.join(__snake_case					,  "pr_oracle.png"    )					,  title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"					,  )
      merge_eval(__snake_case					,  __snake_case					,  "pr_exact"    )
      merge_eval(__snake_case					,  __snake_case					,  "pr_f1"    )
      merge_eval(__snake_case					,  __snake_case					,  "pr_oracle"    )







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )     ->     List[Any]:






      '''simple docstring'''







      if not qid_list:
            return
      UpperCAmelCase_  :	int				      =		[na_probs[k] for k in qid_list]
      UpperCAmelCase_  :	List[Any]				      =		np.ones_like(__snake_case    ) / float(len(__snake_case    )    )
      plt.hist(__snake_case					,  weights=__snake_case					,  bins=2_0					,  range=(0.0, 1.0)    )
      plt.xlabel("Model probability of no-answer"    )
      plt.ylabel("Proportion of dataset"    )
      plt.title(F"""Histogram of no-answer probability: {name}"""    )
      plt.savefig(os.path.join(__snake_case					,  F"""na_prob_hist_{name}.png"""    )    )
      plt.clf()







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )     ->     List[str]:






      '''simple docstring'''







      UpperCAmelCase_  :	Any				      =		sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]    )
      UpperCAmelCase_  :	Optional[Any]				      =		num_no_ans
      UpperCAmelCase_  :	Any				      =		cur_score
      UpperCAmelCase_  :	str				      =		0.0
      UpperCAmelCase_  :	str				      =		sorted(__snake_case					,  key=lambda __snake_case    : na_probs[k]    )
      for i, qid in enumerate(__snake_case    ):
            if qid not in scores:
                  continue
            if qid_to_has_ans[qid]:
                  UpperCAmelCase_  :	Optional[Any]				      =		scores[qid]
            else:
                  if preds[qid]:
                        UpperCAmelCase_  :	Optional[Any]				      =		-1
                  else:
                        UpperCAmelCase_  :	List[Any]				      =		0
            cur_score += diff
            if cur_score > best_score:
                  UpperCAmelCase_  :	Tuple				      =		cur_score
                  UpperCAmelCase_  :	Optional[Any]				      =		na_probs[qid]
      return 100.0 * best_score / len(__snake_case    ), best_thresh







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )     ->     Optional[Any]:






      '''simple docstring'''







      UpperCAmelCase_      ,      UpperCAmelCase_  :	Any				      =		find_best_thresh(__snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )
      UpperCAmelCase_      ,      UpperCAmelCase_  :	str				      =		find_best_thresh(__snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )
      UpperCAmelCase_  :	Tuple				      =		best_exact
      UpperCAmelCase_  :	Optional[int]				      =		exact_thresh
      UpperCAmelCase_  :	Optional[int]				      =		best_fa
      UpperCAmelCase_  :	Tuple				      =		fa_thresh







def    _snake_case (       )     ->     Optional[int]:






      '''simple docstring'''







      with open(OPTS.data_file    ) as f:
            UpperCAmelCase_  :	Any				      =		json.load(__snake_case    )
            UpperCAmelCase_  :	Optional[Any]				      =		dataset_json["data"]
      with open(OPTS.pred_file    ) as f:
            UpperCAmelCase_  :	List[str]				      =		json.load(__snake_case    )
      if OPTS.na_prob_file:
            with open(OPTS.na_prob_file    ) as f:
                  UpperCAmelCase_  :	Dict				      =		json.load(__snake_case    )
      else:
            UpperCAmelCase_  :	Dict				      =		{k: 0.0 for k in preds}
      UpperCAmelCase_  :	Optional[int]				      =		make_qid_to_has_ans(__snake_case    )  # maps qid to True/False
      UpperCAmelCase_  :	Dict				      =		[k for k, v in qid_to_has_ans.items() if v]
      UpperCAmelCase_  :	Optional[int]				      =		[k for k, v in qid_to_has_ans.items() if not v]
      UpperCAmelCase_      ,      UpperCAmelCase_  :	Union[str, Any]				      =		get_raw_scores(__snake_case					,  __snake_case    )
      UpperCAmelCase_  :	Tuple				      =		apply_no_ans_threshold(__snake_case					,  __snake_case					,  __snake_case					,  OPTS.na_prob_thresh    )
      UpperCAmelCase_  :	Optional[int]				      =		apply_no_ans_threshold(__snake_case					,  __snake_case					,  __snake_case					,  OPTS.na_prob_thresh    )
      UpperCAmelCase_  :	Optional[Any]				      =		make_eval_dict(__snake_case					,  __snake_case    )
      if has_ans_qids:
            UpperCAmelCase_  :	int				      =		make_eval_dict(__snake_case					,  __snake_case					,  qid_list=__snake_case    )
            merge_eval(__snake_case					,  __snake_case					,  "HasAns"    )
      if no_ans_qids:
            UpperCAmelCase_  :	List[str]				      =		make_eval_dict(__snake_case					,  __snake_case					,  qid_list=__snake_case    )
            merge_eval(__snake_case					,  __snake_case					,  "NoAns"    )
      if OPTS.na_prob_file:
            find_all_best_thresh(__snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )
      if OPTS.na_prob_file and OPTS.out_image_dir:
            run_precision_recall_analysis(__snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  __snake_case					,  OPTS.out_image_dir    )
            histogram_na_prob(__snake_case					,  __snake_case					,  OPTS.out_image_dir					,  "hasAns"    )
            histogram_na_prob(__snake_case					,  __snake_case					,  OPTS.out_image_dir					,  "noAns"    )
      if OPTS.out_file:
            with open(OPTS.out_file					,  "w"    ) as f:
                  json.dump(__snake_case					,  __snake_case    )
      else:
            print(json.dumps(__snake_case					,  indent=2    )    )


if __name__ == "__main__":
   __lowerCamelCase         =			parse_args()
   if OPTS.out_image_dir:
      import matplotlib

      matplotlib.use('''Agg''')
      import matplotlib.pyplot as plt
   main()





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase         =			logging.get_logger(__name__)

__lowerCamelCase         =			{
    '''microsoft/trocr-base-handwritten''': (
        '''https://huggingface.co/microsoft/trocr-base-handwritten/resolve/main/config.json'''
    ),
    # See all TrOCR models at https://huggingface.co/models?filter=trocr
}


class   snake_case_ (lowercase__  ):




       """simple docstring"""






       _lowerCamelCase			    =				"""trocr"""
       _lowerCamelCase			    =				["""past_key_values"""]
       _lowerCamelCase			    =				{
           """num_attention_heads""": """decoder_attention_heads""",
           """hidden_size""": """d_model""",
           """num_hidden_layers""": """decoder_layers""",
       }



       def __init__(     self     ,lowercase=50265     ,lowercase=1024     ,lowercase=12     ,lowercase=16     ,lowercase=4096     ,lowercase="gelu"     ,lowercase=512     ,lowercase=0.1     ,lowercase=0.0     ,lowercase=0.0     ,lowercase=2     ,lowercase=0.02     ,lowercase=0.0     ,lowercase=True     ,lowercase=False     ,lowercase=True     ,lowercase=True     ,lowercase=1     ,lowercase=0     ,lowercase=2     ,**lowercase     ,):

             """simple docstring"""





             UpperCAmelCase_  :	int				      =		vocab_size
             UpperCAmelCase_  :	Tuple				      =		d_model
             UpperCAmelCase_  :	List[str]				      =		decoder_layers
             UpperCAmelCase_  :	Union[str, Any]				      =		decoder_attention_heads
             UpperCAmelCase_  :	Optional[Any]				      =		decoder_ffn_dim
             UpperCAmelCase_  :	Tuple				      =		activation_function
             UpperCAmelCase_  :	List[Any]				      =		max_position_embeddings
             UpperCAmelCase_  :	List[Any]				      =		dropout
             UpperCAmelCase_  :	List[str]				      =		attention_dropout
             UpperCAmelCase_  :	Union[str, Any]				      =		activation_dropout
             UpperCAmelCase_  :	Optional[int]				      =		init_std
             UpperCAmelCase_  :	Dict				      =		decoder_layerdrop
             UpperCAmelCase_  :	List[str]				      =		use_cache
             UpperCAmelCase_  :	Dict				      =		scale_embedding
             UpperCAmelCase_  :	Dict				      =		use_learned_position_embeddings
             UpperCAmelCase_  :	Optional[Any]				      =		layernorm_embedding

             super().__init__(
                 pad_token_id=lowercase     ,bos_token_id=lowercase     ,eos_token_id=lowercase     ,decoder_start_token_id=lowercase     ,**lowercase     ,)
