




import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_UpperCAmelCase	:						Optional[int]			=     re.compile(r"\b(a|an|the)\b", re.UNICODE)

_UpperCAmelCase	:						Any			=     None



def       UpperCAmelCase__  (     ):
  lowercase    :Optional[int]      	=						argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."   )
  parser.add_argument("data_file",			metavar="data.json",			help="Input data JSON file."   )
  parser.add_argument("pred_file",			metavar="pred.json",			help="Model predictions."   )
  parser.add_argument(
      "--out-file",			"-o",			metavar="eval.json",			help="Write accuracy metrics to file (default is stdout)."   )
  parser.add_argument(
      "--na-prob-file",			"-n",			metavar="na_prob.json",			help="Model estimates of probability of no answer."   )
  parser.add_argument(
      "--na-prob-thresh",			"-t",			type=lowerCamelCase,			default=1.0,			help="Predict \"\" if no-answer probability exceeds this (default = 1.0).",			)
  parser.add_argument(
      "--out-image-dir",			"-p",			metavar="out_images",			default=lowerCamelCase,			help="Save precision-recall curves to directory."   )
  parser.add_argument("--verbose",			"-v",			action="store_true"   )
  if len(sys.argv   ) == 1:
    parser.print_help()
    sys.exit(1   )
  return parser.parse_args()



def       UpperCAmelCase__  (     lowerCamelCase   ):
  lowercase    :str      	=						{}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        lowercase    :List[str]      	=						bool(qa["answers"]["text"]   )
  return qid_to_has_ans



def       UpperCAmelCase__  (     lowerCamelCase   ):

  def remove_articles(lowerCamelCase   ):
    return ARTICLES_REGEX.sub(" ",			lowerCamelCase   )

  def white_space_fix(lowerCamelCase   ):
    return " ".join(text.split()   )

  def remove_punc(lowerCamelCase   ):
    lowercase    :Optional[int]      	=						set(string.punctuation   )
    return "".join(ch for ch in text if ch not in exclude   )

  def lower(lowerCamelCase   ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(lowerCamelCase   )   )   )   )



def       UpperCAmelCase__  (     lowerCamelCase   ):
  if not s:
    return []
  return normalize_answer(lowerCamelCase   ).split()



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase   ):
  return int(normalize_answer(lowerCamelCase   ) == normalize_answer(lowerCamelCase   )   )



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase   ):
  lowercase    :List[Any]      	=						get_tokens(lowerCamelCase   )
  lowercase    :Tuple      	=						get_tokens(lowerCamelCase   )
  lowercase    :List[str]      	=						collections.Counter(lowerCamelCase   ) & collections.Counter(lowerCamelCase   )
  lowercase    :List[Any]      	=						sum(common.values()   )
  if len(lowerCamelCase   ) == 0 or len(lowerCamelCase   ) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks   )
  if num_same == 0:
    return 0
  lowercase    :Tuple      	=						1.0 * num_same / len(lowerCamelCase   )
  lowercase    :Tuple      	=						1.0 * num_same / len(lowerCamelCase   )
  lowercase    :Union[str, Any]      	=						(2 * precision * recall) / (precision + recall)
  return fa



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase   ):
  lowercase    :Optional[int]      	=						{}
  lowercase    :Optional[int]      	=						{}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        lowercase    :Optional[Any]      	=						qa["id"]
        lowercase    :str      	=						[t for t in qa["answers"]["text"] if normalize_answer(lowerCamelCase   )]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          lowercase    :int      	=						[""]
        if qid not in preds:
          print(F"Missing prediction for {qid}"   )
          continue
        lowercase    :List[str]      	=						preds[qid]
        # Take max over all gold answers
        lowercase    :Optional[Any]      	=						max(compute_exact(lowerCamelCase,			lowerCamelCase   ) for a in gold_answers   )
        lowercase    :Dict      	=						max(compute_fa(lowerCamelCase,			lowerCamelCase   ) for a in gold_answers   )
  return exact_scores, fa_scores



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   ):
  lowercase    :List[Any]      	=						{}
  for qid, s in scores.items():
    lowercase    :Any      	=						na_probs[qid] > na_prob_thresh
    if pred_na:
      lowercase    :Optional[int]      	=						float(not qid_to_has_ans[qid]   )
    else:
      lowercase    :List[Any]      	=						s
  return new_scores



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase=None   ):
  if not qid_list:
    lowercase    :Optional[Any]      	=						len(lowerCamelCase   )
    return collections.OrderedDict(
        [
            ("exact", 100.0 * sum(exact_scores.values()   ) / total),
            ("f1", 100.0 * sum(fa_scores.values()   ) / total),
            ("total", total),
        ]   )
  else:
    lowercase    :str      	=						len(lowerCamelCase   )
    return collections.OrderedDict(
        [
            ("exact", 100.0 * sum(exact_scores[k] for k in qid_list   ) / total),
            ("f1", 100.0 * sum(fa_scores[k] for k in qid_list   ) / total),
            ("total", total),
        ]   )



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase   ):
  for k in new_eval:
    lowercase    :List[Any]      	=						new_eval[k]



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   ):
  plt.step(lowerCamelCase,			lowerCamelCase,			color="b",			alpha=0.2,			where="post"   )
  plt.fill_between(lowerCamelCase,			lowerCamelCase,			step="post",			alpha=0.2,			color="b"   )
  plt.xlabel("Recall"   )
  plt.ylabel("Precision"   )
  plt.xlim([0.0, 1.05]   )
  plt.ylim([0.0, 1.05]   )
  plt.title(lowerCamelCase   )
  plt.savefig(lowerCamelCase   )
  plt.clf()



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase=None,			lowerCamelCase=None   ):
  lowercase    :Union[str, Any]      	=						sorted(lowerCamelCase,			key=lambda lowerCamelCase   : na_probs[k]   )
  lowercase    :int      	=						0.0
  lowercase    :Dict      	=						1.0
  lowercase    :Dict      	=						0.0
  lowercase    :Optional[int]      	=						[1.0]
  lowercase    :int      	=						[0.0]
  lowercase    :str      	=						0.0
  for i, qid in enumerate(lowerCamelCase   ):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    lowercase    :List[Any]      	=						true_pos / float(i + 1   )
    lowercase    :List[Any]      	=						true_pos / float(lowerCamelCase   )
    if i == len(lowerCamelCase   ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(lowerCamelCase   )
      recalls.append(lowerCamelCase   )
  if out_image:
    plot_pr_curve(lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   )
  return {"ap": 100.0 * avg_prec}



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   ):
  if out_image_dir and not os.path.exists(lowerCamelCase   ):
    os.makedirs(lowerCamelCase   )
  lowercase    :Optional[Any]      	=						sum(1 for v in qid_to_has_ans.values() if v   )
  if num_true_pos == 0:
    return
  lowercase    :str      	=						make_precision_recall_eval(
      lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			out_image=os.path.join(lowerCamelCase,			"pr_exact.png"   ),			title="Precision-Recall curve for Exact Match score",			)
  lowercase    :Dict      	=						make_precision_recall_eval(
      lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			out_image=os.path.join(lowerCamelCase,			"pr_f1.png"   ),			title="Precision-Recall curve for F1 score",			)
  lowercase    :Optional[int]      	=						{k: float(lowerCamelCase   ) for k, v in qid_to_has_ans.items()}
  lowercase    :str      	=						make_precision_recall_eval(
      lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			out_image=os.path.join(lowerCamelCase,			"pr_oracle.png"   ),			title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)",			)
  merge_eval(lowerCamelCase,			lowerCamelCase,			"pr_exact"   )
  merge_eval(lowerCamelCase,			lowerCamelCase,			"pr_f1"   )
  merge_eval(lowerCamelCase,			lowerCamelCase,			"pr_oracle"   )



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   ):
  if not qid_list:
    return
  lowercase    :Optional[int]      	=						[na_probs[k] for k in qid_list]
  lowercase    :int      	=						np.ones_like(lowerCamelCase   ) / float(len(lowerCamelCase   )   )
  plt.hist(lowerCamelCase,			weights=lowerCamelCase,			bins=20,			range=(0.0, 1.0)   )
  plt.xlabel("Model probability of no-answer"   )
  plt.ylabel("Proportion of dataset"   )
  plt.title(F"Histogram of no-answer probability: {name}"   )
  plt.savefig(os.path.join(lowerCamelCase,			F"na_prob_hist_{name}.png"   )   )
  plt.clf()



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   ):
  lowercase    :int      	=						sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]   )
  lowercase    :Dict      	=						num_no_ans
  lowercase    :int      	=						cur_score
  lowercase    :List[str]      	=						0.0
  lowercase    :Dict      	=						sorted(lowerCamelCase,			key=lambda lowerCamelCase   : na_probs[k]   )
  for i, qid in enumerate(lowerCamelCase   ):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      lowercase    :Any      	=						scores[qid]
    else:
      if preds[qid]:
        lowercase    :List[str]      	=						-1
      else:
        lowercase    :Optional[Any]      	=						0
    cur_score += diff
    if cur_score > best_score:
      lowercase    :Any      	=						cur_score
      lowercase    :Optional[Any]      	=						na_probs[qid]
  return 100.0 * best_score / len(lowerCamelCase   ), best_thresh



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   ):
  lowercase	,    lowercase    :List[str]      	=						find_best_thresh(lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   )
  lowercase	,    lowercase    :List[Any]      	=						find_best_thresh(lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   )
  lowercase    :Tuple      	=						best_exact
  lowercase    :Optional[int]      	=						exact_thresh
  lowercase    :Tuple      	=						best_fa
  lowercase    :List[str]      	=						fa_thresh



def       UpperCAmelCase__  (     ):
  with open(OPTS.data_file   ) as f:
    lowercase    :Any      	=						json.load(lowerCamelCase   )
    lowercase    :int      	=						dataset_json["data"]
  with open(OPTS.pred_file   ) as f:
    lowercase    :Tuple      	=						json.load(lowerCamelCase   )
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file   ) as f:
      lowercase    :str      	=						json.load(lowerCamelCase   )
  else:
    lowercase    :Optional[int]      	=						{k: 0.0 for k in preds}
  lowercase    :Tuple      	=						make_qid_to_has_ans(lowerCamelCase   )  # maps qid to True/False
  lowercase    :str      	=						[k for k, v in qid_to_has_ans.items() if v]
  lowercase    :Optional[int]      	=						[k for k, v in qid_to_has_ans.items() if not v]
  lowercase	,    lowercase    :Any      	=						get_raw_scores(lowerCamelCase,			lowerCamelCase   )
  lowercase    :List[Any]      	=						apply_no_ans_threshold(lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			OPTS.na_prob_thresh   )
  lowercase    :Optional[Any]      	=						apply_no_ans_threshold(lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			OPTS.na_prob_thresh   )
  lowercase    :int      	=						make_eval_dict(lowerCamelCase,			lowerCamelCase   )
  if has_ans_qids:
    lowercase    :str      	=						make_eval_dict(lowerCamelCase,			lowerCamelCase,			qid_list=lowerCamelCase   )
    merge_eval(lowerCamelCase,			lowerCamelCase,			"HasAns"   )
  if no_ans_qids:
    lowercase    :int      	=						make_eval_dict(lowerCamelCase,			lowerCamelCase,			qid_list=lowerCamelCase   )
    merge_eval(lowerCamelCase,			lowerCamelCase,			"NoAns"   )
  if OPTS.na_prob_file:
    find_all_best_thresh(lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase   )
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			lowerCamelCase,			OPTS.out_image_dir   )
    histogram_na_prob(lowerCamelCase,			lowerCamelCase,			OPTS.out_image_dir,			"hasAns"   )
    histogram_na_prob(lowerCamelCase,			lowerCamelCase,			OPTS.out_image_dir,			"noAns"   )
  if OPTS.out_file:
    with open(OPTS.out_file,			"w"   ) as f:
      json.dump(lowerCamelCase,			lowerCamelCase   )
  else:
    print(json.dumps(lowerCamelCase,			indent=2   )   )


if __name__ == "__main__":
     _UpperCAmelCase	:						Optional[Any]			=     parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use("Agg")
          import matplotlib.pyplot as plt
     main()









import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType




class    __lowerCAmelCase       (							lowerCAmelCase):
       _a 					=						['''image_processor''', '''tokenizer''']
       _a 					=						'''LayoutLMv2ImageProcessor'''
       _a 					=						('''LayoutXLMTokenizer''', '''LayoutXLMTokenizerFast''')






       def __init__(   self:					int       ,					_lowerCAmelCase:					Any=None       ,					_lowerCAmelCase:					List[Any]=None       ,					**_lowerCAmelCase:					Optional[Any]				):
         if "feature_extractor" in kwargs:
           warnings.warn(
               "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
               " instead."       ,					_lowerCAmelCase       ,					)
           lowercase    :Optional[Any]      	=						kwargs.pop("feature_extractor"				)

         lowercase    :Union[str, Any]      	=						image_processor if image_processor is not None else feature_extractor
         if image_processor is None:
           raise ValueError("You need to specify an `image_processor`."				)
         if tokenizer is None:
           raise ValueError("You need to specify a `tokenizer`."				)

         super().__init__(_lowerCAmelCase       ,					_lowerCAmelCase				)






       def __call__(   self:					List[str]       ,					_lowerCAmelCase:					Optional[Any]       ,					_lowerCAmelCase:					Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None       ,					_lowerCAmelCase:					Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None       ,					_lowerCAmelCase:					Union[List[List[int]], List[List[List[int]]]] = None       ,					_lowerCAmelCase:					Optional[Union[List[int], List[List[int]]]] = None       ,					_lowerCAmelCase:					bool = True       ,					_lowerCAmelCase:					Union[bool, str, PaddingStrategy] = False       ,					_lowerCAmelCase:					Union[bool, str, TruncationStrategy] = None       ,					_lowerCAmelCase:					Optional[int] = None       ,					_lowerCAmelCase:					int = 0       ,					_lowerCAmelCase:					Optional[int] = None       ,					_lowerCAmelCase:					Optional[bool] = None       ,					_lowerCAmelCase:					Optional[bool] = None       ,					_lowerCAmelCase:					bool = False       ,					_lowerCAmelCase:					bool = False       ,					_lowerCAmelCase:					bool = False       ,					_lowerCAmelCase:					bool = False       ,					_lowerCAmelCase:					bool = True       ,					_lowerCAmelCase:					Optional[Union[str, TensorType]] = None       ,					**_lowerCAmelCase:					Tuple       ,					):
         # verify input
         if self.image_processor.apply_ocr and (boxes is not None):
           raise ValueError(
               "You cannot provide bounding boxes "
               "if you initialized the image processor with apply_ocr set to True."				)

         if self.image_processor.apply_ocr and (word_labels is not None):
           raise ValueError(
               "You cannot provide word labels if you initialized the image processor with apply_ocr set to True."				)

         if return_overflowing_tokens is True and return_offsets_mapping is False:
           raise ValueError("You cannot return overflowing tokens without returning the offsets mapping."				)

         # first, apply the image processor
         lowercase    :int      	=						self.image_processor(images=_lowerCAmelCase       ,					return_tensors=_lowerCAmelCase				)

         # second, apply the tokenizer
         if text is not None and self.image_processor.apply_ocr and text_pair is None:
           if isinstance(_lowerCAmelCase       ,					_lowerCAmelCase				):
             lowercase    :int      	=						[text]  # add batch dimension (as the image processor always adds a batch dimension)
           lowercase    :int      	=						features["words"]

         lowercase    :Optional[Any]      	=						self.tokenizer(
             text=text if text is not None else features["words"]       ,					text_pair=text_pair if text_pair is not None else None       ,					boxes=boxes if boxes is not None else features["boxes"]       ,					word_labels=_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase       ,					padding=_lowerCAmelCase       ,					truncation=_lowerCAmelCase       ,					max_length=_lowerCAmelCase       ,					stride=_lowerCAmelCase       ,					pad_to_multiple_of=_lowerCAmelCase       ,					return_token_type_ids=_lowerCAmelCase       ,					return_attention_mask=_lowerCAmelCase       ,					return_overflowing_tokens=_lowerCAmelCase       ,					return_special_tokens_mask=_lowerCAmelCase       ,					return_offsets_mapping=_lowerCAmelCase       ,					return_length=_lowerCAmelCase       ,					verbose=_lowerCAmelCase       ,					return_tensors=_lowerCAmelCase       ,					**_lowerCAmelCase       ,					)

         # add pixel values
         lowercase    :Any      	=						features.pop("pixel_values"				)
         if return_overflowing_tokens is True:
           lowercase    :Any      	=						self.get_overflowing_images(_lowerCAmelCase       ,					encoded_inputs["overflow_to_sample_mapping"]				)
         lowercase    :Optional[int]      	=						images

         return encoded_inputs






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple       ,					_lowerCAmelCase:					Tuple       ,					_lowerCAmelCase:					List[Any]				):
         # in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
         lowercase    :Dict      	=						[]
         for sample_idx in overflow_to_sample_mapping:
           images_with_overflow.append(images[sample_idx]				)

         if len(_lowerCAmelCase				) != len(_lowerCAmelCase				):
           raise ValueError(
               "Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"
               F" {len(_lowerCAmelCase				)} and {len(_lowerCAmelCase				)}"				)

         return images_with_overflow






       def 					SCREAMING_SNAKE_CASE    (   self:					Dict       ,					*_lowerCAmelCase:					str       ,					**_lowerCAmelCase:					List[Any]				):
         return self.tokenizer.batch_decode(*_lowerCAmelCase       ,					**_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					str       ,					*_lowerCAmelCase:					Tuple       ,					**_lowerCAmelCase:					Union[str, Any]				):
         return self.tokenizer.decode(*_lowerCAmelCase       ,					**_lowerCAmelCase				)






       @property
       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         return ["input_ids", "bbox", "attention_mask", "image"]






       @property
       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]				):
         warnings.warn(
             "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."       ,					_lowerCAmelCase       ,					)
         return self.image_processor_class






       @property
       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         warnings.warn(
             "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."       ,					_lowerCAmelCase       ,					)
         return self.image_processor



