


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
       import torch

if is_vision_available():
       from PIL import Image

       from transformers import ChineseCLIPImageProcessor


class   a__	(						unittest.TestCase    ):

       def __init__(   self      :     List[Any]      ,							lowerCamelCase_      :     Dict      ,							lowerCamelCase_      :     Optional[int]=7      ,							lowerCamelCase_      :     int=3      ,							lowerCamelCase_      :     List[Any]=1_8      ,							lowerCamelCase_      :     Dict=3_0      ,							lowerCamelCase_      :     Optional[int]=4_0_0      ,							lowerCamelCase_      :     Union[str, Any]=True      ,							lowerCamelCase_      :     Union[str, Any]=None      ,							lowerCamelCase_      :     Optional[int]=True      ,							lowerCamelCase_      :     int=None      ,							lowerCamelCase_      :     Optional[Any]=True      ,							lowerCamelCase_      :     Tuple=[0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3]      ,							lowerCamelCase_      :     int=[0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1]      ,							lowerCamelCase_      :     List[str]=True      ,							):
            a_    :     Dict										=    size if size is not None else {"""height""": 2_2_4, """width""": 2_2_4}
            a_    :     int										=    crop_size if crop_size is not None else {"""height""": 1_8, """width""": 1_8}
            a_    :     str										=    parent
            a_    :     str										=    batch_size
            a_    :     Dict										=    num_channels
            a_    :     int										=    image_size
            a_    :     Any										=    min_resolution
            a_    :     List[Any]										=    max_resolution
            a_    :     List[str]										=    do_resize
            a_    :     Any										=    size
            a_    :     int										=    do_center_crop
            a_    :     Dict										=    crop_size
            a_    :     List[str]										=    do_normalize
            a_    :     Union[str, Any]										=    image_mean
            a_    :     Optional[int]										=    image_std
            a_    :     List[str]										=    do_convert_rgb

       def 						UpperCAmelCase(   self      :     Optional[int]    ):
            return {
                "do_resize": self.do_resize,
                "size": self.size,
                "do_center_crop": self.do_center_crop,
                "crop_size": self.crop_size,
                "do_normalize": self.do_normalize,
                "image_mean": self.image_mean,
                "image_std": self.image_std,
                "do_convert_rgb": self.do_convert_rgb,
            }

       def 						UpperCAmelCase(   self      :     int      ,							lowerCamelCase_      :     Union[str, Any]=False      ,							lowerCamelCase_      :     Dict=False      ,							lowerCamelCase_      :     Tuple=False    ):
            assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

            if equal_resolution:
                 a_    :     Tuple										=    []
                 for i in range(self.batch_size    ):
                      image_inputs.append(
                          np.random.randint(
                              2_5_5      ,							size=(self.num_channels, self.max_resolution, self.max_resolution)      ,							dtype=np.uinta    )    )
            else:
                 a_    :     List[str]										=    []
                 for i in range(self.batch_size    ):
                      a_			,			a_    :     Any										=    np.random.choice(np.arange(self.min_resolution      ,							self.max_resolution    )      ,							2    )
                      image_inputs.append(np.random.randint(2_5_5      ,							size=(self.num_channels, width, height)      ,							dtype=np.uinta    )    )

            if not numpify and not torchify:
                 # PIL expects the channel dimension as last dimension
                 a_    :     Optional[int]										=    [Image.fromarray(np.moveaxis(lowerCamelCase_      ,							0      ,							-1    )    ) for x in image_inputs]

            if torchify:
                 a_    :     Optional[int]										=    [torch.from_numpy(lowerCamelCase_    ) for x in image_inputs]

            return image_inputs


@require_torch
@require_vision
class   a__	(						UpperCamelCase__						,       unittest.TestCase    ):
       lowerCamelCase__:     Any							    =     ChineseCLIPImageProcessor if is_vision_available() else None

       def 						UpperCAmelCase(   self      :     Tuple    ):
            a_    :     str										=    ChineseCLIPImageProcessingTester(self      ,							do_center_crop=lowerCamelCase_    )

       @property
       def 						UpperCAmelCase(   self      :     int    ):
            return self.image_processor_tester.prepare_image_processor_dict()

       def 						UpperCAmelCase(   self      :     Any    ):
            a_    :     str										=    self.image_processing_class(**self.image_processor_dict    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""do_resize"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""size"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""do_center_crop"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""center_crop"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""do_normalize"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""image_mean"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""image_std"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""do_convert_rgb"""    )    )

       def 						UpperCAmelCase(   self      :     str    ):
            a_    :     Tuple										=    self.image_processing_class.from_dict(self.image_processor_dict    )
            self.assertEqual(image_processor.size      ,							{"""height""": 2_2_4, """width""": 2_2_4}    )
            self.assertEqual(image_processor.crop_size      ,							{"""height""": 1_8, """width""": 1_8}    )

            a_    :     int										=    self.image_processing_class.from_dict(self.image_processor_dict      ,							size=4_2      ,							crop_size=8_4    )
            self.assertEqual(image_processor.size      ,							{"""shortest_edge""": 4_2}    )
            self.assertEqual(image_processor.crop_size      ,							{"""height""": 8_4, """width""": 8_4}    )

       def 						UpperCAmelCase(   self      :     List[Any]    ):
            pass

       def 						UpperCAmelCase(   self      :     List[str]    ):
            # Initialize image_processing
            a_    :     Tuple										=    self.image_processing_class(**self.image_processor_dict    )
            # create random PIL images
            a_    :     str										=    self.image_processor_tester.prepare_inputs(equal_resolution=lowerCamelCase_    )
            for image in image_inputs:
                 self.assertIsInstance(lowerCamelCase_      ,							Image.Image    )

            # Test not batched input
            a_    :     Dict										=    image_processing(image_inputs[0]      ,							return_tensors="""pt"""    ).pixel_values
            self.assertEqual(
                encoded_images.shape      ,							(
                    1,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size["""height"""],
                    self.image_processor_tester.crop_size["""width"""],
                )      ,							)

            # Test batched
            a_    :     List[Any]										=    image_processing(lowerCamelCase_      ,							return_tensors="""pt"""    ).pixel_values
            self.assertEqual(
                encoded_images.shape      ,							(
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size["""height"""],
                    self.image_processor_tester.crop_size["""width"""],
                )      ,							)

       def 						UpperCAmelCase(   self      :     int    ):
            # Initialize image_processing
            a_    :     str										=    self.image_processing_class(**self.image_processor_dict    )
            # create random numpy tensors
            a_    :     Dict										=    self.image_processor_tester.prepare_inputs(equal_resolution=lowerCamelCase_      ,							numpify=lowerCamelCase_    )
            for image in image_inputs:
                 self.assertIsInstance(lowerCamelCase_      ,							np.ndarray    )

            # Test not batched input
            a_    :     Optional[int]										=    image_processing(image_inputs[0]      ,							return_tensors="""pt"""    ).pixel_values
            self.assertEqual(
                encoded_images.shape      ,							(
                    1,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size["""height"""],
                    self.image_processor_tester.crop_size["""width"""],
                )      ,							)

            # Test batched
            a_    :     Any										=    image_processing(lowerCamelCase_      ,							return_tensors="""pt"""    ).pixel_values
            self.assertEqual(
                encoded_images.shape      ,							(
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size["""height"""],
                    self.image_processor_tester.crop_size["""width"""],
                )      ,							)

       def 						UpperCAmelCase(   self      :     Any    ):
            # Initialize image_processing
            a_    :     Any										=    self.image_processing_class(**self.image_processor_dict    )
            # create random PyTorch tensors
            a_    :     Dict										=    self.image_processor_tester.prepare_inputs(equal_resolution=lowerCamelCase_      ,							torchify=lowerCamelCase_    )
            for image in image_inputs:
                 self.assertIsInstance(lowerCamelCase_      ,							torch.Tensor    )

            # Test not batched input
            a_    :     List[str]										=    image_processing(image_inputs[0]      ,							return_tensors="""pt"""    ).pixel_values
            self.assertEqual(
                encoded_images.shape      ,							(
                    1,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size["""height"""],
                    self.image_processor_tester.crop_size["""width"""],
                )      ,							)

            # Test batched
            a_    :     int										=    image_processing(lowerCamelCase_      ,							return_tensors="""pt"""    ).pixel_values
            self.assertEqual(
                encoded_images.shape      ,							(
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size["""height"""],
                    self.image_processor_tester.crop_size["""width"""],
                )      ,							)


@require_torch
@require_vision
class   a__	(						UpperCamelCase__						,       unittest.TestCase    ):
       lowerCamelCase__:     int							    =     ChineseCLIPImageProcessor if is_vision_available() else None

       def 						UpperCAmelCase(   self      :     str    ):
            a_    :     List[str]										=    ChineseCLIPImageProcessingTester(self      ,							num_channels=4      ,							do_center_crop=lowerCamelCase_    )
            a_    :     Optional[int]										=    3

       @property
       def 						UpperCAmelCase(   self      :     Optional[int]    ):
            return self.image_processor_tester.prepare_image_processor_dict()

       def 						UpperCAmelCase(   self      :     Optional[int]    ):
            a_    :     str										=    self.image_processing_class(**self.image_processor_dict    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""do_resize"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""size"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""do_center_crop"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""center_crop"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""do_normalize"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""image_mean"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""image_std"""    )    )
            self.assertTrue(hasattr(lowerCamelCase_      ,							"""do_convert_rgb"""    )    )

       def 						UpperCAmelCase(   self      :     List[Any]    ):
            pass

       def 						UpperCAmelCase(   self      :     Tuple    ):
            # Initialize image_processing
            a_    :     List[str]										=    self.image_processing_class(**self.image_processor_dict    )
            # create random PIL images
            a_    :     Optional[Any]										=    self.image_processor_tester.prepare_inputs(equal_resolution=lowerCamelCase_    )
            for image in image_inputs:
                 self.assertIsInstance(lowerCamelCase_      ,							Image.Image    )

            # Test not batched input
            a_    :     List[str]										=    image_processing(image_inputs[0]      ,							return_tensors="""pt"""    ).pixel_values
            self.assertEqual(
                encoded_images.shape      ,							(
                    1,
                    self.expected_encoded_image_num_channels,
                    self.image_processor_tester.crop_size["""height"""],
                    self.image_processor_tester.crop_size["""width"""],
                )      ,							)

            # Test batched
            a_    :     Union[str, Any]										=    image_processing(lowerCamelCase_      ,							return_tensors="""pt"""    ).pixel_values
            self.assertEqual(
                encoded_images.shape      ,							(
                    self.image_processor_tester.batch_size,
                    self.expected_encoded_image_num_channels,
                    self.image_processor_tester.crop_size["""height"""],
                    self.image_processor_tester.crop_size["""width"""],
                )      ,							)








import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__lowerCamelCase        =     re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

__lowerCamelCase        =     None



def  _a			(						):
     a_    :     Tuple										=    argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0.""" )
     parser.add_argument("""data_file"""  ,  metavar="""data.json"""  ,  help="""Input data JSON file.""" )
     parser.add_argument("""pred_file"""  ,  metavar="""pred.json"""  ,  help="""Model predictions.""" )
     parser.add_argument(
         """--out-file"""  ,  """-o"""  ,  metavar="""eval.json"""  ,  help="""Write accuracy metrics to file (default is stdout).""" )
     parser.add_argument(
         """--na-prob-file"""  ,  """-n"""  ,  metavar="""na_prob.json"""  ,  help="""Model estimates of probability of no answer.""" )
     parser.add_argument(
         """--na-prob-thresh"""  ,  """-t"""  ,  type=__UpperCamelCase  ,  default=1.0  ,  help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""  ,  )
     parser.add_argument(
         """--out-image-dir"""  ,  """-p"""  ,  metavar="""out_images"""  ,  default=__UpperCamelCase  ,  help="""Save precision-recall curves to directory.""" )
     parser.add_argument("""--verbose"""  ,  """-v"""  ,  action="""store_true""" )
     if len(sys.argv ) == 1:
          parser.print_help()
          sys.exit(1 )
     return parser.parse_args()



def  _a			(						__UpperCamelCase ):
     a_    :     Union[str, Any]										=    {}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a_    :     Union[str, Any]										=    bool(qa["""answers"""]["""text"""] )
     return qid_to_has_ans



def  _a			(						__UpperCamelCase ):

     def remove_articles(__UpperCamelCase ):
          return ARTICLES_REGEX.sub(""" """  ,  __UpperCamelCase )

     def white_space_fix(__UpperCamelCase ):
          return " ".join(text.split() )

     def remove_punc(__UpperCamelCase ):
          a_    :     List[str]										=    set(string.punctuation )
          return "".join(ch for ch in text if ch not in exclude )

     def lower(__UpperCamelCase ):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(__UpperCamelCase ) ) ) )



def  _a			(						__UpperCamelCase ):
     if not s:
          return []
     return normalize_answer(__UpperCamelCase ).split()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     return int(normalize_answer(__UpperCamelCase ) == normalize_answer(__UpperCamelCase ) )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     List[Any]										=    get_tokens(__UpperCamelCase )
     a_    :     str										=    get_tokens(__UpperCamelCase )
     a_    :     Tuple										=    collections.Counter(__UpperCamelCase ) & collections.Counter(__UpperCamelCase )
     a_    :     str										=    sum(common.values() )
     if len(__UpperCamelCase ) == 0 or len(__UpperCamelCase ) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks )
     if num_same == 0:
          return 0
     a_    :     Any										=    1.0 * num_same / len(__UpperCamelCase )
     a_    :     Optional[int]										=    1.0 * num_same / len(__UpperCamelCase )
     a_    :     List[Any]										=    (2 * precision * recall) / (precision + recall)
     return fa



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Tuple										=    {}
     a_    :     Optional[int]										=    {}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a_    :     str										=    qa["""id"""]
                    a_    :     Dict										=    [t for t in qa["""answers"""]["""text"""] if normalize_answer(__UpperCamelCase )]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         a_    :     Union[str, Any]										=    [""""""]
                    if qid not in preds:
                         print(F'''Missing prediction for {qid}''' )
                         continue
                    a_    :     int										=    preds[qid]
                    # Take max over all gold answers
                    a_    :     Union[str, Any]										=    max(compute_exact(__UpperCamelCase  ,  __UpperCamelCase ) for a in gold_answers )
                    a_    :     int										=    max(compute_fa(__UpperCamelCase  ,  __UpperCamelCase ) for a in gold_answers )
     return exact_scores, fa_scores



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Optional[Any]										=    {}
     for qid, s in scores.items():
          a_    :     Optional[int]										=    na_probs[qid] > na_prob_thresh
          if pred_na:
               a_    :     List[Any]										=    float(not qid_to_has_ans[qid] )
          else:
               a_    :     List[Any]										=    s
     return new_scores



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase=None ):
     if not qid_list:
          a_    :     List[Any]										=    len(__UpperCamelCase )
          return collections.OrderedDict(
              [
                  ("""exact""", 1_00.0 * sum(exact_scores.values() ) / total),
                  ("""f1""", 1_00.0 * sum(fa_scores.values() ) / total),
                  ("""total""", total),
              ] )
     else:
          a_    :     Tuple										=    len(__UpperCamelCase )
          return collections.OrderedDict(
              [
                  ("""exact""", 1_00.0 * sum(exact_scores[k] for k in qid_list ) / total),
                  ("""f1""", 1_00.0 * sum(fa_scores[k] for k in qid_list ) / total),
                  ("""total""", total),
              ] )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     for k in new_eval:
          a_    :     Any										=    new_eval[k]



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     plt.step(__UpperCamelCase  ,  __UpperCamelCase  ,  color="""b"""  ,  alpha=0.2  ,  where="""post""" )
     plt.fill_between(__UpperCamelCase  ,  __UpperCamelCase  ,  step="""post"""  ,  alpha=0.2  ,  color="""b""" )
     plt.xlabel("""Recall""" )
     plt.ylabel("""Precision""" )
     plt.xlim([0.0, 1.05] )
     plt.ylim([0.0, 1.05] )
     plt.title(__UpperCamelCase )
     plt.savefig(__UpperCamelCase )
     plt.clf()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase=None  ,  __UpperCamelCase=None ):
     a_    :     List[Any]										=    sorted(__UpperCamelCase  ,  key=lambda __UpperCamelCase : na_probs[k] )
     a_    :     int										=    0.0
     a_    :     Tuple										=    1.0
     a_    :     Union[str, Any]										=    0.0
     a_    :     List[str]										=    [1.0]
     a_    :     int										=    [0.0]
     a_    :     Optional[int]										=    0.0
     for i, qid in enumerate(__UpperCamelCase ):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          a_    :     Union[str, Any]										=    true_pos / float(i + 1 )
          a_    :     Tuple										=    true_pos / float(__UpperCamelCase )
          if i == len(__UpperCamelCase ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(__UpperCamelCase )
               recalls.append(__UpperCamelCase )
     if out_image:
          plot_pr_curve(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     return {"ap": 1_00.0 * avg_prec}



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     if out_image_dir and not os.path.exists(__UpperCamelCase ):
          os.makedirs(__UpperCamelCase )
     a_    :     Any										=    sum(1 for v in qid_to_has_ans.values() if v )
     if num_true_pos == 0:
          return
     a_    :     Union[str, Any]										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_exact.png""" )  ,  title="""Precision-Recall curve for Exact Match score"""  ,  )
     a_    :     Any										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_f1.png""" )  ,  title="""Precision-Recall curve for F1 score"""  ,  )
     a_    :     int										=    {k: float(__UpperCamelCase ) for k, v in qid_to_has_ans.items()}
     a_    :     Dict										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_oracle.png""" )  ,  title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""  ,  )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_exact""" )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_f1""" )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_oracle""" )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     if not qid_list:
          return
     a_    :     List[Any]										=    [na_probs[k] for k in qid_list]
     a_    :     List[Any]										=    np.ones_like(__UpperCamelCase ) / float(len(__UpperCamelCase ) )
     plt.hist(__UpperCamelCase  ,  weights=__UpperCamelCase  ,  bins=2_0  ,  range=(0.0, 1.0) )
     plt.xlabel("""Model probability of no-answer""" )
     plt.ylabel("""Proportion of dataset""" )
     plt.title(F'''Histogram of no-answer probability: {name}''' )
     plt.savefig(os.path.join(__UpperCamelCase  ,  F'''na_prob_hist_{name}.png''' ) )
     plt.clf()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Optional[int]										=    sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k] )
     a_    :     Tuple										=    num_no_ans
     a_    :     List[Any]										=    cur_score
     a_    :     List[Any]										=    0.0
     a_    :     int										=    sorted(__UpperCamelCase  ,  key=lambda __UpperCamelCase : na_probs[k] )
     for i, qid in enumerate(__UpperCamelCase ):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               a_    :     Optional[Any]										=    scores[qid]
          else:
               if preds[qid]:
                    a_    :     Optional[int]										=    -1
               else:
                    a_    :     Optional[int]										=    0
          cur_score += diff
          if cur_score > best_score:
               a_    :     str										=    cur_score
               a_    :     int										=    na_probs[qid]
     return 1_00.0 * best_score / len(__UpperCamelCase ), best_thresh



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_			,			a_    :     Optional[int]										=    find_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     a_			,			a_    :     Optional[Any]										=    find_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     a_    :     Dict										=    best_exact
     a_    :     Optional[Any]										=    exact_thresh
     a_    :     Optional[Any]										=    best_fa
     a_    :     Union[str, Any]										=    fa_thresh



def  _a			(						):
     with open(OPTS.data_file ) as f:
          a_    :     str										=    json.load(__UpperCamelCase )
          a_    :     Optional[int]										=    dataset_json["""data"""]
     with open(OPTS.pred_file ) as f:
          a_    :     Dict										=    json.load(__UpperCamelCase )
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file ) as f:
               a_    :     List[str]										=    json.load(__UpperCamelCase )
     else:
          a_    :     List[str]										=    {k: 0.0 for k in preds}
     a_    :     Any										=    make_qid_to_has_ans(__UpperCamelCase )  # maps qid to True/False
     a_    :     Any										=    [k for k, v in qid_to_has_ans.items() if v]
     a_    :     Union[str, Any]										=    [k for k, v in qid_to_has_ans.items() if not v]
     a_			,			a_    :     List[str]										=    get_raw_scores(__UpperCamelCase  ,  __UpperCamelCase )
     a_    :     int										=    apply_no_ans_threshold(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.na_prob_thresh )
     a_    :     Optional[int]										=    apply_no_ans_threshold(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.na_prob_thresh )
     a_    :     List[str]										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase )
     if has_ans_qids:
          a_    :     Dict										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase  ,  qid_list=__UpperCamelCase )
          merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """HasAns""" )
     if no_ans_qids:
          a_    :     List[str]										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase  ,  qid_list=__UpperCamelCase )
          merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """NoAns""" )
     if OPTS.na_prob_file:
          find_all_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir )
          histogram_na_prob(__UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir  ,  """hasAns""" )
          histogram_na_prob(__UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir  ,  """noAns""" )
     if OPTS.out_file:
          with open(OPTS.out_file  ,  """w""" ) as f:
               json.dump(__UpperCamelCase  ,  __UpperCamelCase )
     else:
          print(json.dumps(__UpperCamelCase  ,  indent=2 ) )


if __name__ == "__main__":
       __lowerCamelCase        =     parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('''Agg''')
              import matplotlib.pyplot as plt
       main()



