'''simple docstring'''

import inspect
import unittest

from transformers import YolosConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch
  from torch import nn

  from transformers import YolosForObjectDetection, YolosModel
  from transformers.models.yolos.modeling_yolos import YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
  from PIL import Image

  from transformers import AutoImageProcessor


class 				_lowercase	:
      '''simple docstring'''


      def __init__(				self					,lowerCamelCase_					,lowerCamelCase_=13					,lowerCamelCase_=[30, 30]					,lowerCamelCase_=2					,lowerCamelCase_=3					,lowerCamelCase_=True					,lowerCamelCase_=True					,lowerCamelCase_=32					,lowerCamelCase_=5					,lowerCamelCase_=4					,lowerCamelCase_=37					,lowerCamelCase_="gelu"					,lowerCamelCase_=0.1					,lowerCamelCase_=0.1					,lowerCamelCase_=10					,lowerCamelCase_=0.02					,lowerCamelCase_=3					,lowerCamelCase_=None					,lowerCamelCase_=8					,lowerCamelCase_=10					,)			->			str:



         '''simple docstring'''



         UpperCAmelCase__							:				Tuple   				=  parent
         UpperCAmelCase__							:				Optional[int]   				=  batch_size
         UpperCAmelCase__							:				Tuple   				=  image_size
         UpperCAmelCase__							:				int   				=  patch_size
         UpperCAmelCase__							:				Optional[int]   				=  num_channels
         UpperCAmelCase__							:				Any   				=  is_training
         UpperCAmelCase__							:				Tuple   				=  use_labels
         UpperCAmelCase__							:				List[Any]   				=  hidden_size
         UpperCAmelCase__							:				Any   				=  num_hidden_layers
         UpperCAmelCase__							:				List[str]   				=  num_attention_heads
         UpperCAmelCase__							:				Optional[int]   				=  intermediate_size
         UpperCAmelCase__							:				Union[str, Any]   				=  hidden_act
         UpperCAmelCase__							:				Any   				=  hidden_dropout_prob
         UpperCAmelCase__							:				str   				=  attention_probs_dropout_prob
         UpperCAmelCase__							:				Union[str, Any]   				=  type_sequence_label_size
         UpperCAmelCase__							:				int   				=  initializer_range
         UpperCAmelCase__							:				Optional[int]   				=  num_labels
         UpperCAmelCase__							:				Optional[int]   				=  scope
         UpperCAmelCase__							:				Union[str, Any]   				=  n_targets
         UpperCAmelCase__							:				int   				=  num_detection_tokens
         # we set the expected sequence length (which is used in several tests)
         # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token) + num_detection_tokens
         UpperCAmelCase__							:				int   				=  (image_size[1] // patch_size) * (image_size[0] // patch_size)
         UpperCAmelCase__							:				List[Any]   				=  num_patches + 1 + self.num_detection_tokens




      def 		lowerCAmelCase__					(				self							)			->			str:



         '''simple docstring'''



         UpperCAmelCase__							:				Dict   				=  floats_tensor([self.batch_size, self.num_channels, self.image_size[0], self.image_size[1]]							)

         UpperCAmelCase__							:				List[str]   				=  None
         if self.use_labels:
            # labels is a list of Dict (each Dict being the labels for a given example in the batch)
            UpperCAmelCase__							:				List[Any]   				=  []
            for i in range(self.batch_size							):
               UpperCAmelCase__							:				Any   				=  {}
               UpperCAmelCase__							:				str   				=  torch.randint(
                   high=self.num_labels					,size=(self.n_targets,)					,device=lowerCamelCase_							)
               UpperCAmelCase__							:				int   				=  torch.rand(self.n_targets					,4					,device=lowerCamelCase_							)
               labels.append(lowerCamelCase_							)

         UpperCAmelCase__							:				Any   				=  self.get_config()

         return config, pixel_values, labels




      def 		lowerCAmelCase__					(				self							)			->			Any:



         '''simple docstring'''



         return YolosConfig(
             image_size=self.image_size					,patch_size=self.patch_size					,num_channels=self.num_channels					,hidden_size=self.hidden_size					,num_hidden_layers=self.num_hidden_layers					,num_attention_heads=self.num_attention_heads					,intermediate_size=self.intermediate_size					,hidden_act=self.hidden_act					,hidden_dropout_prob=self.hidden_dropout_prob					,attention_probs_dropout_prob=self.attention_probs_dropout_prob					,is_decoder=lowerCamelCase_					,initializer_range=self.initializer_range					,num_detection_tokens=self.num_detection_tokens					,num_labels=self.num_labels					,)




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_					,lowerCamelCase_					,lowerCamelCase_							)			->			Tuple:



         '''simple docstring'''



         UpperCAmelCase__							:				int   				=  YolosModel(config=lowerCamelCase_							)
         model.to(lowerCamelCase_							)
         model.eval()
         UpperCAmelCase__							:				Optional[int]   				=  model(lowerCamelCase_							)
         self.parent.assertEqual(
             result.last_hidden_state.shape					,(self.batch_size, self.expected_seq_len, self.hidden_size)							)




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_					,lowerCamelCase_					,lowerCamelCase_							)			->			Any:



         '''simple docstring'''



         UpperCAmelCase__							:				Optional[Any]   				=  YolosForObjectDetection(lowerCamelCase_							)
         model.to(lowerCamelCase_							)
         model.eval()

         UpperCAmelCase__							:				Dict   				=  model(pixel_values=lowerCamelCase_							)
         UpperCAmelCase__							:				str   				=  model(lowerCamelCase_							)

         self.parent.assertEqual(result.logits.shape					,(self.batch_size, self.num_detection_tokens, self.num_labels + 1)							)
         self.parent.assertEqual(result.pred_boxes.shape					,(self.batch_size, self.num_detection_tokens, 4)							)

         UpperCAmelCase__							:				Any   				=  model(pixel_values=lowerCamelCase_					,labels=lowerCamelCase_							)

         self.parent.assertEqual(result.loss.shape					,()							)
         self.parent.assertEqual(result.logits.shape					,(self.batch_size, self.num_detection_tokens, self.num_labels + 1)							)
         self.parent.assertEqual(result.pred_boxes.shape					,(self.batch_size, self.num_detection_tokens, 4)							)




      def 		lowerCAmelCase__					(				self							)			->			Any:



         '''simple docstring'''



         UpperCAmelCase__							:				List[Any]   				=  self.prepare_config_and_inputs()
         UpperCAmelCase__				,      UpperCAmelCase__				,      UpperCAmelCase__							:				str   				=  config_and_inputs
         UpperCAmelCase__							:				str   				=  {'''pixel_values''': pixel_values}
         return config, inputs_dict




@require_torch
class 				_lowercase	(      lowerCAmelCase		,lowerCAmelCase		,unittest.TestCase   ):
      '''simple docstring'''


      UpperCAmelCase_      :      Any      							=   (YolosModel, YolosForObjectDetection) if is_torch_available() else ()
      UpperCAmelCase_      :      str      							=   (
          {'''feature-extraction''': YolosModel, '''object-detection''': YolosForObjectDetection} if is_torch_available() else {}
      )

      UpperCAmelCase_      :      List[Any]      							=   False
      UpperCAmelCase_      :      str      							=   False
      UpperCAmelCase_      :      str      							=   False
      UpperCAmelCase_      :      Optional[Any]      							=   False




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_					,lowerCamelCase_					,lowerCamelCase_=False							)			->			List[Any]:



         '''simple docstring'''



         UpperCAmelCase__							:				int   				=  super()._prepare_for_class(lowerCamelCase_					,lowerCamelCase_					,return_labels=lowerCamelCase_							)

         if return_labels:
            if model_class.__name__ == "YolosForObjectDetection":
               UpperCAmelCase__							:				int   				=  []
               for i in range(self.model_tester.batch_size							):
                  UpperCAmelCase__							:				List[str]   				=  {}
                  UpperCAmelCase__							:				Optional[int]   				=  torch.ones(
                      size=(self.model_tester.n_targets,)					,device=lowerCamelCase_					,dtype=torch.long							)
                  UpperCAmelCase__							:				Optional[int]   				=  torch.ones(
                      self.model_tester.n_targets					,4					,device=lowerCamelCase_					,dtype=torch.float							)
                  labels.append(lowerCamelCase_							)
               UpperCAmelCase__							:				List[str]   				=  labels

         return inputs_dict




      def 		lowerCAmelCase__					(				self							)			->			int:



         '''simple docstring'''



         UpperCAmelCase__							:				int   				=  YolosModelTester(self							)
         UpperCAmelCase__							:				Union[str, Any]   				=  ConfigTester(self					,config_class=lowerCamelCase_					,has_text_modality=lowerCamelCase_					,hidden_size=37							)




      def 		lowerCAmelCase__					(				self							)			->			List[Any]:



         '''simple docstring'''



         self.config_tester.run_common_tests()




      def 		lowerCAmelCase__					(				self							)			->			str:



         '''simple docstring'''



         pass




      def 		lowerCAmelCase__					(				self							)			->			List[Any]:



         '''simple docstring'''



         UpperCAmelCase__				,      UpperCAmelCase__							:				Tuple   				=  self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            UpperCAmelCase__							:				Dict   				=  model_class(lowerCamelCase_							)
            self.assertIsInstance(model.get_input_embeddings()					,(nn.Module)							)
            UpperCAmelCase__							:				Union[str, Any]   				=  model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(lowerCamelCase_					,nn.Linear							)							)




      def 		lowerCAmelCase__					(				self							)			->			List[str]:



         '''simple docstring'''



         UpperCAmelCase__				,      UpperCAmelCase__							:				Union[str, Any]   				=  self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            UpperCAmelCase__							:				int   				=  model_class(lowerCamelCase_							)
            UpperCAmelCase__							:				Any   				=  inspect.signature(model.forward							)
            # signature.parameters is an OrderedDict => so arg_names order is deterministic
            UpperCAmelCase__							:				int   				=  [*signature.parameters.keys()]

            UpperCAmelCase__							:				int   				=  ['''pixel_values''']
            self.assertListEqual(arg_names[:1]					,lowerCamelCase_							)




      def 		lowerCAmelCase__					(				self							)			->			List[str]:



         '''simple docstring'''



         UpperCAmelCase__							:				int   				=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*lowerCamelCase_							)




      def 		lowerCAmelCase__					(				self							)			->			List[str]:



         '''simple docstring'''



         UpperCAmelCase__				,      UpperCAmelCase__							:				Union[str, Any]   				=  self.model_tester.prepare_config_and_inputs_for_common()
         UpperCAmelCase__							:				Optional[Any]   				=  True

         # in YOLOS, the seq_len is different
         UpperCAmelCase__							:				Dict   				=  self.model_tester.expected_seq_len
         for model_class in self.all_model_classes:
            UpperCAmelCase__							:				List[Any]   				=  True
            UpperCAmelCase__							:				List[Any]   				=  False
            UpperCAmelCase__							:				Optional[Any]   				=  True
            UpperCAmelCase__							:				str   				=  model_class(lowerCamelCase_							)
            model.to(lowerCamelCase_							)
            model.eval()
            with torch.no_grad():
               UpperCAmelCase__							:				List[str]   				=  model(**self._prepare_for_class(lowerCamelCase_					,lowerCamelCase_							)							)
            UpperCAmelCase__							:				Any   				=  outputs.attentions
            self.assertEqual(len(lowerCamelCase_							)					,self.model_tester.num_hidden_layers							)

            # check that output_attentions also work using config
            del inputs_dict["output_attentions"]
            UpperCAmelCase__							:				List[Any]   				=  True
            UpperCAmelCase__							:				Any   				=  model_class(lowerCamelCase_							)
            model.to(lowerCamelCase_							)
            model.eval()
            with torch.no_grad():
               UpperCAmelCase__							:				Optional[int]   				=  model(**self._prepare_for_class(lowerCamelCase_					,lowerCamelCase_							)							)
            UpperCAmelCase__							:				List[Any]   				=  outputs.attentions
            self.assertEqual(len(lowerCamelCase_							)					,self.model_tester.num_hidden_layers							)

            self.assertListEqual(
                list(attentions[0].shape[-3:]							)					,[self.model_tester.num_attention_heads, seq_len, seq_len]					,)
            UpperCAmelCase__							:				str   				=  len(lowerCamelCase_							)

            # Check attention is always last and order is fine
            UpperCAmelCase__							:				Tuple   				=  True
            UpperCAmelCase__							:				Tuple   				=  True
            UpperCAmelCase__							:				str   				=  model_class(lowerCamelCase_							)
            model.to(lowerCamelCase_							)
            model.eval()
            with torch.no_grad():
               UpperCAmelCase__							:				int   				=  model(**self._prepare_for_class(lowerCamelCase_					,lowerCamelCase_							)							)

            UpperCAmelCase__							:				Dict   				=  1
            self.assertEqual(out_len + added_hidden_states					,len(lowerCamelCase_							)							)

            UpperCAmelCase__							:				int   				=  outputs.attentions

            self.assertEqual(len(lowerCamelCase_							)					,self.model_tester.num_hidden_layers							)
            self.assertListEqual(
                list(self_attentions[0].shape[-3:]							)					,[self.model_tester.num_attention_heads, seq_len, seq_len]					,)




      def 		lowerCAmelCase__					(				self							)			->			List[str]:



         '''simple docstring'''



         def check_hidden_states_output(lowerCamelCase_					,lowerCamelCase_					,lowerCamelCase_							):
            UpperCAmelCase__							:				List[str]   				=  model_class(lowerCamelCase_							)
            model.to(lowerCamelCase_							)
            model.eval()

            with torch.no_grad():
               UpperCAmelCase__							:				Optional[int]   				=  model(**self._prepare_for_class(lowerCamelCase_					,lowerCamelCase_							)							)

            UpperCAmelCase__							:				Dict   				=  outputs.hidden_states

            UpperCAmelCase__							:				Tuple   				=  getattr(
                self.model_tester					,'''expected_num_hidden_layers'''					,self.model_tester.num_hidden_layers + 1							)
            self.assertEqual(len(lowerCamelCase_							)					,lowerCamelCase_							)

            # YOLOS has a different seq_length
            UpperCAmelCase__							:				Optional[int]   				=  self.model_tester.expected_seq_len

            self.assertListEqual(
                list(hidden_states[0].shape[-2:]							)					,[seq_length, self.model_tester.hidden_size]					,)

         UpperCAmelCase__				,      UpperCAmelCase__							:				str   				=  self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            UpperCAmelCase__							:				Union[str, Any]   				=  True
            check_hidden_states_output(lowerCamelCase_					,lowerCamelCase_					,lowerCamelCase_							)

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            UpperCAmelCase__							:				Dict   				=  True

            check_hidden_states_output(lowerCamelCase_					,lowerCamelCase_					,lowerCamelCase_							)




      def 		lowerCAmelCase__					(				self							)			->			Dict:



         '''simple docstring'''



         UpperCAmelCase__							:				Tuple   				=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_object_detection(*lowerCamelCase_							)




      @slow
      def 		lowerCAmelCase__					(				self							)			->			Dict:



         '''simple docstring'''



         for model_name in YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            UpperCAmelCase__							:				List[str]   				=  YolosModel.from_pretrained(lowerCamelCase_							)
            self.assertIsNotNone(lowerCamelCase_							)







def __UpperCamelCase(   ):
   '''simple docstring'''
   UpperCAmelCase__							:				Dict   				=  Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''    )
   return image


@require_torch
@require_vision
class 				_lowercase	(      unittest.TestCase   ):
      '''simple docstring'''


      @cached_property
      def 		lowerCAmelCase__					(				self							)			->			Dict:



         '''simple docstring'''



         return AutoImageProcessor.from_pretrained('''hustvl/yolos-small'''							) if is_vision_available() else None




      @slow
      def 		lowerCAmelCase__					(				self							)			->			Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__							:				Tuple   				=  YolosForObjectDetection.from_pretrained('''hustvl/yolos-small'''							).to(lowerCamelCase_							)

         UpperCAmelCase__							:				Optional[Any]   				=  self.default_image_processor
         UpperCAmelCase__							:				Any   				=  prepare_img()
         UpperCAmelCase__							:				Any   				=  image_processor(images=lowerCamelCase_					,return_tensors='''pt'''							).to(lowerCamelCase_							)

         # forward pass
         with torch.no_grad():
            UpperCAmelCase__							:				Any   				=  model(inputs.pixel_values							)

         # verify outputs
         UpperCAmelCase__							:				Dict   				=  torch.Size((1, 100, 92)							)
         self.assertEqual(outputs.logits.shape					,lowerCamelCase_							)

         UpperCAmelCase__							:				Optional[Any]   				=  torch.tensor(
             [[-24.0248, -10.3024, -14.8290], [-42.0392, -16.8200, -27.4334], [-27.2743, -11.8154, -18.7148]]					,device=lowerCamelCase_					,)
         UpperCAmelCase__							:				Optional[Any]   				=  torch.tensor(
             [[0.2559, 0.5455, 0.4706], [0.2989, 0.7279, 0.1875], [0.7732, 0.4017, 0.4462]]					,device=lowerCamelCase_							)
         self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]					,lowerCamelCase_					,atol=1e-4							)							)
         self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3]					,lowerCamelCase_					,atol=1e-4							)							)

         # verify postprocessing
         UpperCAmelCase__							:				Union[str, Any]   				=  image_processor.post_process_object_detection(
             lowerCamelCase_					,threshold=0.3					,target_sizes=[image.size[::-1]]							)[0]
         UpperCAmelCase__							:				Optional[Any]   				=  torch.tensor([0.9994, 0.9790, 0.9964, 0.9972, 0.9861]							).to(lowerCamelCase_							)
         UpperCAmelCase__							:				Optional[int]   				=  [75, 75, 17, 63, 17]
         UpperCAmelCase__							:				Any   				=  torch.tensor([335.0609, 79.3848, 375.4216, 187.2495]							).to(lowerCamelCase_							)

         self.assertEqual(len(results['''scores''']							)					,5							)
         self.assertTrue(torch.allclose(results['''scores''']					,lowerCamelCase_					,atol=1e-4							)							)
         self.assertSequenceEqual(results['''labels'''].tolist()					,lowerCamelCase_							)
         self.assertTrue(torch.allclose(results['''boxes'''][0, :]					,lowerCamelCase_							)							)





'''simple docstring'''

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCamelCase__     :	Union[str, Any]				=			re.compile(r'\b(a|an|the)\b', re.UNICODE)

UpperCamelCase__     :	List[Any]				=			None







def __UpperCamelCase(   ):
   '''simple docstring'''
   UpperCAmelCase__							:				Union[str, Any]   				=  argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''    )
   parser.add_argument('''data_file'''  ,			metavar='''data.json'''  ,			help='''Input data JSON file.'''    )
   parser.add_argument('''pred_file'''  ,			metavar='''pred.json'''  ,			help='''Model predictions.'''    )
   parser.add_argument(
       '''--out-file'''  ,			'''-o'''  ,			metavar='''eval.json'''  ,			help='''Write accuracy metrics to file (default is stdout).'''    )
   parser.add_argument(
       '''--na-prob-file'''  ,			'''-n'''  ,			metavar='''na_prob.json'''  ,			help='''Model estimates of probability of no answer.'''    )
   parser.add_argument(
       '''--na-prob-thresh'''  ,			'''-t'''  ,			type=_A  ,			default=1.0  ,			help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''  ,			)
   parser.add_argument(
       '''--out-image-dir'''  ,			'''-p'''  ,			metavar='''out_images'''  ,			default=_A  ,			help='''Save precision-recall curves to directory.'''    )
   parser.add_argument('''--verbose'''  ,			'''-v'''  ,			action='''store_true'''    )
   if len(sys.argv    ) == 1:
      parser.print_help()
      sys.exit(1    )
   return parser.parse_args()







def __UpperCamelCase(   _A					:				Union[str, Any]    ):
   '''simple docstring'''
   UpperCAmelCase__							:				List[str]   				=  {}
   for article in dataset:
      for p in article["paragraphs"]:
         for qa in p["qas"]:
            UpperCAmelCase__							:				Union[str, Any]   				=  bool(qa['''answers''']['''text''']    )
   return qid_to_has_ans







def __UpperCamelCase(   _A					:				Dict    ):
   '''simple docstring'''

   def remove_articles(_A					:				Union[str, Any]    ):
      return ARTICLES_REGEX.sub(''' '''  ,			_A    )

   def white_space_fix(_A					:				Optional[int]    ):
      return " ".join(text.split()    )

   def remove_punc(_A					:				Optional[Any]    ):
      UpperCAmelCase__							:				Any   				=  set(string.punctuation    )
      return "".join(ch for ch in text if ch not in exclude    )

   def lower(_A					:				Union[str, Any]    ):
      return text.lower()

   return white_space_fix(remove_articles(remove_punc(lower(_A    )    )    )    )







def __UpperCamelCase(   _A					:				Optional[Any]    ):
   '''simple docstring'''
   if not s:
      return []
   return normalize_answer(_A    ).split()







def __UpperCamelCase(   _A					:				Tuple  ,			_A					:				str    ):
   '''simple docstring'''
   return int(normalize_answer(_A    ) == normalize_answer(_A    )    )







def __UpperCamelCase(   _A					:				Optional[Any]  ,			_A					:				Union[str, Any]    ):
   '''simple docstring'''
   UpperCAmelCase__							:				Dict   				=  get_tokens(_A    )
   UpperCAmelCase__							:				Tuple   				=  get_tokens(_A    )
   UpperCAmelCase__							:				Any   				=  collections.Counter(_A    ) & collections.Counter(_A    )
   UpperCAmelCase__							:				List[Any]   				=  sum(common.values()    )
   if len(_A    ) == 0 or len(_A    ) == 0:
      # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
      return int(gold_toks == pred_toks    )
   if num_same == 0:
      return 0
   UpperCAmelCase__							:				Optional[Any]   				=  1.0 * num_same / len(_A    )
   UpperCAmelCase__							:				Tuple   				=  1.0 * num_same / len(_A    )
   UpperCAmelCase__							:				Optional[int]   				=  (2 * precision * recall) / (precision + recall)
   return fa







def __UpperCamelCase(   _A					:				List[str]  ,			_A					:				Optional[int]    ):
   '''simple docstring'''
   UpperCAmelCase__							:				int   				=  {}
   UpperCAmelCase__							:				Any   				=  {}
   for article in dataset:
      for p in article["paragraphs"]:
         for qa in p["qas"]:
            UpperCAmelCase__							:				str   				=  qa['''id''']
            UpperCAmelCase__							:				List[Any]   				=  [t for t in qa['''answers''']['''text'''] if normalize_answer(_A    )]
            if not gold_answers:
               # For unanswerable questions, only correct answer is empty string
               UpperCAmelCase__							:				Tuple   				=  ['''''']
            if qid not in preds:
               print(F'''Missing prediction for {qid}'''    )
               continue
            UpperCAmelCase__							:				Union[str, Any]   				=  preds[qid]
            # Take max over all gold answers
            UpperCAmelCase__							:				List[str]   				=  max(compute_exact(_A  ,			_A    ) for a in gold_answers    )
            UpperCAmelCase__							:				List[str]   				=  max(compute_fa(_A  ,			_A    ) for a in gold_answers    )
   return exact_scores, fa_scores







def __UpperCamelCase(   _A					:				Any  ,			_A					:				Optional[Any]  ,			_A					:				List[str]  ,			_A					:				List[Any]    ):
   '''simple docstring'''
   UpperCAmelCase__							:				Optional[int]   				=  {}
   for qid, s in scores.items():
      UpperCAmelCase__							:				Dict   				=  na_probs[qid] > na_prob_thresh
      if pred_na:
         UpperCAmelCase__							:				Any   				=  float(not qid_to_has_ans[qid]    )
      else:
         UpperCAmelCase__							:				List[str]   				=  s
   return new_scores







def __UpperCamelCase(   _A					:				str  ,			_A					:				Optional[Any]  ,			_A					:				Any=None    ):
   '''simple docstring'''
   if not qid_list:
      UpperCAmelCase__							:				List[Any]   				=  len(_A    )
      return collections.OrderedDict(
          [
              ('''exact''', 1_0_0.0 * sum(exact_scores.values()    ) / total),
              ('''f1''', 1_0_0.0 * sum(fa_scores.values()    ) / total),
              ('''total''', total),
          ]    )
   else:
      UpperCAmelCase__							:				List[str]   				=  len(_A    )
      return collections.OrderedDict(
          [
              ('''exact''', 1_0_0.0 * sum(exact_scores[k] for k in qid_list    ) / total),
              ('''f1''', 1_0_0.0 * sum(fa_scores[k] for k in qid_list    ) / total),
              ('''total''', total),
          ]    )







def __UpperCamelCase(   _A					:				List[str]  ,			_A					:				List[Any]  ,			_A					:				Tuple    ):
   '''simple docstring'''
   for k in new_eval:
      UpperCAmelCase__							:				List[str]   				=  new_eval[k]







def __UpperCamelCase(   _A					:				Tuple  ,			_A					:				Any  ,			_A					:				Optional[int]  ,			_A					:				int    ):
   '''simple docstring'''
   plt.step(_A  ,			_A  ,			color='''b'''  ,			alpha=0.2  ,			where='''post'''    )
   plt.fill_between(_A  ,			_A  ,			step='''post'''  ,			alpha=0.2  ,			color='''b'''    )
   plt.xlabel('''Recall'''    )
   plt.ylabel('''Precision'''    )
   plt.xlim([0.0, 1.0_5]    )
   plt.ylim([0.0, 1.0_5]    )
   plt.title(_A    )
   plt.savefig(_A    )
   plt.clf()







def __UpperCamelCase(   _A					:				Optional[int]  ,			_A					:				Tuple  ,			_A					:				Any  ,			_A					:				Any  ,			_A					:				Tuple=None  ,			_A					:				Union[str, Any]=None    ):
   '''simple docstring'''
   UpperCAmelCase__							:				int   				=  sorted(_A  ,			key=lambda _A    : na_probs[k]    )
   UpperCAmelCase__							:				Tuple   				=  0.0
   UpperCAmelCase__							:				Any   				=  1.0
   UpperCAmelCase__							:				Any   				=  0.0
   UpperCAmelCase__							:				Union[str, Any]   				=  [1.0]
   UpperCAmelCase__							:				int   				=  [0.0]
   UpperCAmelCase__							:				Optional[Any]   				=  0.0
   for i, qid in enumerate(_A    ):
      if qid_to_has_ans[qid]:
         true_pos += scores[qid]
      UpperCAmelCase__							:				Optional[Any]   				=  true_pos / float(i + 1    )
      UpperCAmelCase__							:				int   				=  true_pos / float(_A    )
      if i == len(_A    ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
         # i.e., if we can put a threshold after this point
         avg_prec += cur_p * (cur_r - recalls[-1])
         precisions.append(_A    )
         recalls.append(_A    )
   if out_image:
      plot_pr_curve(_A  ,			_A  ,			_A  ,			_A    )
   return {"ap": 1_0_0.0 * avg_prec}







def __UpperCamelCase(   _A					:				Any  ,			_A					:				Optional[Any]  ,			_A					:				List[Any]  ,			_A					:				Any  ,			_A					:				Dict  ,			_A					:				Any    ):
   '''simple docstring'''
   if out_image_dir and not os.path.exists(_A    ):
      os.makedirs(_A    )
   UpperCAmelCase__							:				List[str]   				=  sum(1 for v in qid_to_has_ans.values() if v    )
   if num_true_pos == 0:
      return
   UpperCAmelCase__							:				Dict   				=  make_precision_recall_eval(
       _A  ,			_A  ,			_A  ,			_A  ,			out_image=os.path.join(_A  ,			'''pr_exact.png'''    )  ,			title='''Precision-Recall curve for Exact Match score'''  ,			)
   UpperCAmelCase__							:				Any   				=  make_precision_recall_eval(
       _A  ,			_A  ,			_A  ,			_A  ,			out_image=os.path.join(_A  ,			'''pr_f1.png'''    )  ,			title='''Precision-Recall curve for F1 score'''  ,			)
   UpperCAmelCase__							:				Tuple   				=  {k: float(_A    ) for k, v in qid_to_has_ans.items()}
   UpperCAmelCase__							:				Any   				=  make_precision_recall_eval(
       _A  ,			_A  ,			_A  ,			_A  ,			out_image=os.path.join(_A  ,			'''pr_oracle.png'''    )  ,			title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''  ,			)
   merge_eval(_A  ,			_A  ,			'''pr_exact'''    )
   merge_eval(_A  ,			_A  ,			'''pr_f1'''    )
   merge_eval(_A  ,			_A  ,			'''pr_oracle'''    )







def __UpperCamelCase(   _A					:				Tuple  ,			_A					:				Dict  ,			_A					:				Dict  ,			_A					:				Tuple    ):
   '''simple docstring'''
   if not qid_list:
      return
   UpperCAmelCase__							:				Optional[Any]   				=  [na_probs[k] for k in qid_list]
   UpperCAmelCase__							:				Union[str, Any]   				=  np.ones_like(_A    ) / float(len(_A    )    )
   plt.hist(_A  ,			weights=_A  ,			bins=20  ,			range=(0.0, 1.0)    )
   plt.xlabel('''Model probability of no-answer'''    )
   plt.ylabel('''Proportion of dataset'''    )
   plt.title(F'''Histogram of no-answer probability: {name}'''    )
   plt.savefig(os.path.join(_A  ,			F'''na_prob_hist_{name}.png'''    )    )
   plt.clf()







def __UpperCamelCase(   _A					:				List[Any]  ,			_A					:				List[str]  ,			_A					:				Optional[int]  ,			_A					:				Tuple    ):
   '''simple docstring'''
   UpperCAmelCase__							:				Tuple   				=  sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]    )
   UpperCAmelCase__							:				List[str]   				=  num_no_ans
   UpperCAmelCase__							:				Any   				=  cur_score
   UpperCAmelCase__							:				List[str]   				=  0.0
   UpperCAmelCase__							:				Dict   				=  sorted(_A  ,			key=lambda _A    : na_probs[k]    )
   for i, qid in enumerate(_A    ):
      if qid not in scores:
         continue
      if qid_to_has_ans[qid]:
         UpperCAmelCase__							:				int   				=  scores[qid]
      else:
         if preds[qid]:
            UpperCAmelCase__							:				Any   				=  -1
         else:
            UpperCAmelCase__							:				Dict   				=  0
      cur_score += diff
      if cur_score > best_score:
         UpperCAmelCase__							:				Optional[Any]   				=  cur_score
         UpperCAmelCase__							:				Tuple   				=  na_probs[qid]
   return 1_0_0.0 * best_score / len(_A    ), best_thresh







def __UpperCamelCase(   _A					:				str  ,			_A					:				str  ,			_A					:				int  ,			_A					:				int  ,			_A					:				Tuple  ,			_A					:				Optional[int]    ):
   '''simple docstring'''
   UpperCAmelCase__				,      UpperCAmelCase__							:				Tuple   				=  find_best_thresh(_A  ,			_A  ,			_A  ,			_A    )
   UpperCAmelCase__				,      UpperCAmelCase__							:				Tuple   				=  find_best_thresh(_A  ,			_A  ,			_A  ,			_A    )
   UpperCAmelCase__							:				List[str]   				=  best_exact
   UpperCAmelCase__							:				Any   				=  exact_thresh
   UpperCAmelCase__							:				Dict   				=  best_fa
   UpperCAmelCase__							:				Dict   				=  fa_thresh







def __UpperCamelCase(   ):
   '''simple docstring'''
   with open(OPTS.data_file    ) as f:
      UpperCAmelCase__							:				Dict   				=  json.load(_A    )
      UpperCAmelCase__							:				str   				=  dataset_json['''data''']
   with open(OPTS.pred_file    ) as f:
      UpperCAmelCase__							:				Any   				=  json.load(_A    )
   if OPTS.na_prob_file:
      with open(OPTS.na_prob_file    ) as f:
         UpperCAmelCase__							:				Optional[Any]   				=  json.load(_A    )
   else:
      UpperCAmelCase__							:				Dict   				=  {k: 0.0 for k in preds}
   UpperCAmelCase__							:				int   				=  make_qid_to_has_ans(_A    )  # maps qid to True/False
   UpperCAmelCase__							:				Any   				=  [k for k, v in qid_to_has_ans.items() if v]
   UpperCAmelCase__							:				Optional[Any]   				=  [k for k, v in qid_to_has_ans.items() if not v]
   UpperCAmelCase__				,      UpperCAmelCase__							:				Union[str, Any]   				=  get_raw_scores(_A  ,			_A    )
   UpperCAmelCase__							:				Optional[int]   				=  apply_no_ans_threshold(_A  ,			_A  ,			_A  ,			OPTS.na_prob_thresh    )
   UpperCAmelCase__							:				Dict   				=  apply_no_ans_threshold(_A  ,			_A  ,			_A  ,			OPTS.na_prob_thresh    )
   UpperCAmelCase__							:				Union[str, Any]   				=  make_eval_dict(_A  ,			_A    )
   if has_ans_qids:
      UpperCAmelCase__							:				Optional[int]   				=  make_eval_dict(_A  ,			_A  ,			qid_list=_A    )
      merge_eval(_A  ,			_A  ,			'''HasAns'''    )
   if no_ans_qids:
      UpperCAmelCase__							:				Dict   				=  make_eval_dict(_A  ,			_A  ,			qid_list=_A    )
      merge_eval(_A  ,			_A  ,			'''NoAns'''    )
   if OPTS.na_prob_file:
      find_all_best_thresh(_A  ,			_A  ,			_A  ,			_A  ,			_A  ,			_A    )
   if OPTS.na_prob_file and OPTS.out_image_dir:
      run_precision_recall_analysis(_A  ,			_A  ,			_A  ,			_A  ,			_A  ,			OPTS.out_image_dir    )
      histogram_na_prob(_A  ,			_A  ,			OPTS.out_image_dir  ,			'''hasAns'''    )
      histogram_na_prob(_A  ,			_A  ,			OPTS.out_image_dir  ,			'''noAns'''    )
   if OPTS.out_file:
      with open(OPTS.out_file  ,			'''w'''    ) as f:
         json.dump(_A  ,			_A    )
   else:
      print(json.dumps(_A  ,			indent=2    )    )


if __name__ == "__main__":
  UpperCamelCase__     :	Tuple				=			parse_args()
  if OPTS.out_image_dir:
    import matplotlib

    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
  main()




