





import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


UpperCAmelCase_					:   Union[str, Any] 							=						"""\
    @inproceedings{kakwani2020indicnlpsuite,
    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},
    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},
    year={2020},
    booktitle={Findings of EMNLP},
}
"""

UpperCAmelCase_					:   Dict 							=						"""\
    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide
    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.
"""

UpperCAmelCase_					:   Optional[int] 							=						"""
Compute IndicGLUE evaluation metric associated to each IndicGLUE dataset.
Args:
    predictions: list of predictions to score (as int64),
        except for 'cvit-mkb-clsr' where each prediction is a vector (of float32).
    references: list of ground truth labels corresponding to the predictions (as int64),
        except for 'cvit-mkb-clsr' where each reference is a vector (of float32).
Returns: depending on the IndicGLUE subset, one or several of:
    \"accuracy\": Accuracy
    \"f1\": F1 score
    \"precision\": Precision@10
Examples:

    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'wnli')  # 'wnli' or any of [\"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\"]
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'wiki-ner')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'cvit-mkb-clsr')
    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]
    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]
    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'precision@10': 1.0}

"""






def 							_lowerCAmelCase (  _a   :					Union[str, Any]					,   _a   :					Optional[Any]			)	->			int:
     return float((preds == labels).mean()			)






def 							_lowerCAmelCase (  _a   :					str					,   _a   :					Dict			)	->			List[Any]:
     lowerCAmelCase_					:       Tuple    =		simple_accuracy(SCREAMING_SNAKE_CASE_					,   SCREAMING_SNAKE_CASE_			)
     lowerCAmelCase_					:       List[Any]    =		float(fa_score(y_true=SCREAMING_SNAKE_CASE_					,   y_pred=SCREAMING_SNAKE_CASE_			)			)
     return {
         "accuracy": acc,
         "f1": fa,
     }






def 							_lowerCAmelCase (  _a   :					List[Any]					,   _a   :					List[str]			)	->			List[str]:
     lowerCAmelCase_					:       Dict    =		np.array(SCREAMING_SNAKE_CASE_			)
     lowerCAmelCase_					:       Any    =		np.array(SCREAMING_SNAKE_CASE_			)
     lowerCAmelCase_					:       Dict    =		en_sentvecs.shape[0]

     # mean centering
     lowerCAmelCase_					:       Any    =		en_sentvecs - np.mean(SCREAMING_SNAKE_CASE_					,   axis=0			)
     lowerCAmelCase_					:       Union[str, Any]    =		in_sentvecs - np.mean(SCREAMING_SNAKE_CASE_					,   axis=0			)

     lowerCAmelCase_					:       Tuple    =		cdist(SCREAMING_SNAKE_CASE_					,   SCREAMING_SNAKE_CASE_					,   """cosine"""			)
     lowerCAmelCase_					:       Any    =		np.array(range(SCREAMING_SNAKE_CASE_			)			)
     lowerCAmelCase_					:       Optional[int]    =		sim.argsort(axis=1			)[:, :10]
     lowerCAmelCase_					:       int    =		np.any(preds == actual[:, None]					,   axis=1			)
     return float(matches.mean()			)







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,					_KWARGS_DESCRIPTION  )
class      lowercase__    (							datasets.Metric  ):







  def 	UpperCAmelCase__		(	self						):







       if self.config_name not in [
           "wnli",
           "copa",
           "sna",
           "csqa",
           "wstp",
           "inltkh",
           "bbca",
           "cvit-mkb-clsr",
           "iitp-mr",
           "iitp-pr",
           "actsa-sc",
           "md",
           "wiki-ner",
       ]:
            raise KeyError(
                """You should supply a configuration name selected in """
                """[\"wnli\", \"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", """
                """\"cvit-mkb-clsr\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\", """
                """\"wiki-ner\"]"""						)
       return datasets.MetricInfo(
           description=_DESCRIPTION    ,						citation=_CITATION    ,						inputs_description=_KWARGS_DESCRIPTION    ,						features=datasets.Features(
               {
                   """predictions""": datasets.Value("""int64"""						)
                   if self.config_name != """cvit-mkb-clsr"""
                   else datasets.Sequence(datasets.Value("""float32"""						)						),
                   """references""": datasets.Value("""int64"""						)
                   if self.config_name != """cvit-mkb-clsr"""
                   else datasets.Sequence(datasets.Value("""float32"""						)						),
               }						)    ,						codebase_urls=[]    ,						reference_urls=[]    ,						format="""numpy""" if self.config_name != """cvit-mkb-clsr""" else None    ,						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase						):







       if self.config_name == "cvit-mkb-clsr":
            return {"precision@10": precision_at_aa(_lowercase    ,						_lowercase						)}
       elif self.config_name in ["wiki-ner"]:
            return acc_and_fa(_lowercase    ,						_lowercase						)
       elif self.config_name in [
           "wnli",
           "copa",
           "sna",
           "csqa",
           "wstp",
           "inltkh",
           "bbca",
           "iitp-mr",
           "iitp-pr",
           "actsa-sc",
           "md",
       ]:
            return {"accuracy": simple_accuracy(_lowercase    ,						_lowercase						)}
       else:
            raise KeyError(
                """You should supply a configuration name selected in """
                """[\"wnli\", \"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", """
                """\"cvit-mkb-clsr\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\", """
                """\"wiki-ner\"]"""						)












import unittest

from transformers import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING, AutoTokenizer, is_vision_available
from transformers.pipelines import pipeline
from transformers.pipelines.document_question_answering import apply_tesseract
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_detectrona,
    require_pytesseract,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
      from PIL import Image

      from transformers.image_utils import load_image
else:





      class      lowercase__    :







        @staticmethod
        def 	UpperCAmelCase__		(	*_lowercase    ,						**_lowercase						):
             pass






      def 							_lowerCAmelCase (  _a   :					List[str]			)	->			Any:
           return None


# This is a pinned image from a specific revision of a document question answering space, hosted by HuggingFace,
# so we can expect it to be available.
UpperCAmelCase_					:   Optional[Any] 							=						(
    """https://huggingface.co/spaces/impira/docquery/resolve/2f6c96314dc84dfda62d40de9da55f2f5165d403/invoice.png"""
)





@is_pipeline_test
@require_torch
@require_vision
class      lowercase__    (							unittest.TestCase  ):
  __UpperCamelCase	     =  MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING







  @require_pytesseract
  @require_vision
  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       Optional[int]    =		pipeline(
           """document-question-answering"""    ,						model=_lowercase    ,						tokenizer=_lowercase    ,						image_processor=_lowercase						)

       lowerCAmelCase_					:       List[str]    =		INVOICE_URL
       lowerCAmelCase_					:       str    =		list(zip(*apply_tesseract(load_image(_lowercase						)    ,						_lowercase    ,						""""""						)						)						)
       lowerCAmelCase_					:       int    =		"""What is the placebo?"""
       lowerCAmelCase_					:       str    =		[
           {
               """image""": load_image(_lowercase						),
               """question""": question,
           },
           {
               """image""": image,
               """question""": question,
           },
           {
               """image""": image,
               """question""": question,
               """word_boxes""": word_boxes,
           },
       ]
       return dqa_pipeline, examples







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       List[str]    =		dqa_pipeline(_lowercase    ,						top_k=2						)
       self.assertEqual(
           _lowercase    ,						[
               [
                   {"""score""": ANY(_lowercase						), """answer""": ANY(_lowercase						), """start""": ANY(_lowercase						), """end""": ANY(_lowercase						)},
                   {"""score""": ANY(_lowercase						), """answer""": ANY(_lowercase						), """start""": ANY(_lowercase						), """end""": ANY(_lowercase						)},
               ]
           ]
           * 3    ,						)







  @require_torch
  @require_detectrona
  @require_pytesseract
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Tuple    =		pipeline("""document-question-answering"""    ,						model="""hf-internal-testing/tiny-random-layoutlmv2"""						)
       lowerCAmelCase_					:       str    =		INVOICE_URL
       lowerCAmelCase_					:       int    =		"""How many cats are there?"""

       lowerCAmelCase_					:       List[Any]    =		[
           {"""score""": 0.0001, """answer""": """oy 2312/2019""", """start""": 38, """end""": 39},
           {"""score""": 0.0001, """answer""": """oy 2312/2019 DUE""", """start""": 38, """end""": 40},
       ]
       lowerCAmelCase_					:       Optional[Any]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(nested_simplify(_lowercase    ,						decimals=4						)    ,						_lowercase						)

       lowerCAmelCase_					:       Any    =		dqa_pipeline({"""image""": image, """question""": question}    ,						top_k=2						)
       self.assertEqual(nested_simplify(_lowercase    ,						decimals=4						)    ,						_lowercase						)

       # This image does not detect ANY text in it, meaning layoutlmv2 should fail.
       # Empty answer probably
       lowerCAmelCase_					:       List[Any]    =		"""./tests/fixtures/tests_samples/COCO/000000039769.png"""
       lowerCAmelCase_					:       Any    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(_lowercase    ,						[]						)

       # We can optionnally pass directly the words and bounding boxes
       lowerCAmelCase_					:       int    =		"""./tests/fixtures/tests_samples/COCO/000000039769.png"""
       lowerCAmelCase_					:       int    =		[]
       lowerCAmelCase_					:       List[Any]    =		[]
       lowerCAmelCase_					:       List[Any]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						words=_lowercase    ,						boxes=_lowercase    ,						top_k=2						)
       self.assertEqual(_lowercase    ,						[]						)







  @slow
  @require_torch
  @require_detectrona
  @require_pytesseract
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[Any]    =		pipeline(
           """document-question-answering"""    ,						model="""tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa"""    ,						revision="""9977165"""    ,						)
       lowerCAmelCase_					:       Dict    =		INVOICE_URL
       lowerCAmelCase_					:       int    =		"""What is the invoice number?"""

       lowerCAmelCase_					:       Union[str, Any]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9944, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.0009, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)

       lowerCAmelCase_					:       List[Any]    =		dqa_pipeline({"""image""": image, """question""": question}    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9944, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.0009, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)

       lowerCAmelCase_					:       List[str]    =		dqa_pipeline(
           [{"""image""": image, """question""": question}, {"""image""": image, """question""": question}]    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               [
                   {"""score""": 0.9944, """answer""": """us-001""", """start""": 16, """end""": 16},
                   {"""score""": 0.0009, """answer""": """us-001""", """start""": 16, """end""": 16},
               ],
           ]
           * 2    ,						)







  @slow
  @require_torch
  @require_detectrona
  @require_pytesseract
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[int]    =		pipeline(
           """document-question-answering"""    ,						model="""tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa"""    ,						revision="""9977165"""    ,						max_seq_len=50    ,						)
       lowerCAmelCase_					:       List[str]    =		INVOICE_URL
       lowerCAmelCase_					:       Optional[int]    =		"""What is the invoice number?"""

       lowerCAmelCase_					:       List[Any]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9974, """answer""": """1110212019""", """start""": 23, """end""": 23},
               {"""score""": 0.9948, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)

       lowerCAmelCase_					:       Optional[Any]    =		dqa_pipeline({"""image""": image, """question""": question}    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9974, """answer""": """1110212019""", """start""": 23, """end""": 23},
               {"""score""": 0.9948, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)

       lowerCAmelCase_					:       List[Any]    =		dqa_pipeline(
           [{"""image""": image, """question""": question}, {"""image""": image, """question""": question}]    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               [
                   {"""score""": 0.9974, """answer""": """1110212019""", """start""": 23, """end""": 23},
                   {"""score""": 0.9948, """answer""": """us-001""", """start""": 16, """end""": 16},
               ]
           ]
           * 2    ,						)







  @slow
  @require_torch
  @require_pytesseract
  @require_vision
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       int    =		AutoTokenizer.from_pretrained(
           """impira/layoutlm-document-qa"""    ,						revision="""3dc6de3"""    ,						add_prefix_space=_lowercase						)
       lowerCAmelCase_					:       Any    =		pipeline(
           """document-question-answering"""    ,						model="""impira/layoutlm-document-qa"""    ,						tokenizer=_lowercase    ,						revision="""3dc6de3"""    ,						)
       lowerCAmelCase_					:       int    =		INVOICE_URL
       lowerCAmelCase_					:       Union[str, Any]    =		"""What is the invoice number?"""

       lowerCAmelCase_					:       Optional[Any]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.4251, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.0819, """answer""": """1110212019""", """start""": 23, """end""": 23},
           ]    ,						)

       lowerCAmelCase_					:       int    =		dqa_pipeline({"""image""": image, """question""": question}    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.4251, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.0819, """answer""": """1110212019""", """start""": 23, """end""": 23},
           ]    ,						)

       lowerCAmelCase_					:       Union[str, Any]    =		dqa_pipeline(
           [{"""image""": image, """question""": question}, {"""image""": image, """question""": question}]    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               [
                   {"""score""": 0.4251, """answer""": """us-001""", """start""": 16, """end""": 16},
                   {"""score""": 0.0819, """answer""": """1110212019""", """start""": 23, """end""": 23},
               ]
           ]
           * 2    ,						)

       lowerCAmelCase_					:       int    =		list(zip(*apply_tesseract(load_image(_lowercase						)    ,						_lowercase    ,						""""""						)						)						)

       # This model should also work if `image` is set to None
       lowerCAmelCase_					:       List[Any]    =		dqa_pipeline({"""image""": None, """word_boxes""": word_boxes, """question""": question}    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.4251, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.0819, """answer""": """1110212019""", """start""": 23, """end""": 23},
           ]    ,						)







  @slow
  @require_torch
  @require_pytesseract
  @require_vision
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       int    =		AutoTokenizer.from_pretrained(
           """impira/layoutlm-document-qa"""    ,						revision="""3dc6de3"""    ,						add_prefix_space=_lowercase						)
       lowerCAmelCase_					:       Optional[Any]    =		pipeline(
           """document-question-answering"""    ,						model="""impira/layoutlm-document-qa"""    ,						tokenizer=_lowercase    ,						revision="""3dc6de3"""    ,						max_seq_len=50    ,						)
       lowerCAmelCase_					:       Tuple    =		INVOICE_URL
       lowerCAmelCase_					:       Tuple    =		"""What is the invoice number?"""

       lowerCAmelCase_					:       Optional[int]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9999, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.9998, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)

       lowerCAmelCase_					:       int    =		dqa_pipeline(
           [{"""image""": image, """question""": question}, {"""image""": image, """question""": question}]    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               [
                   {"""score""": 0.9999, """answer""": """us-001""", """start""": 16, """end""": 16},
                   {"""score""": 0.9998, """answer""": """us-001""", """start""": 16, """end""": 16},
               ]
           ]
           * 2    ,						)

       lowerCAmelCase_					:       Union[str, Any]    =		list(zip(*apply_tesseract(load_image(_lowercase						)    ,						_lowercase    ,						""""""						)						)						)

       # This model should also work if `image` is set to None
       lowerCAmelCase_					:       List[str]    =		dqa_pipeline({"""image""": None, """word_boxes""": word_boxes, """question""": question}    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9999, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.9998, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)







  @slow
  @require_torch
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       int    =		pipeline(
           """document-question-answering"""    ,						model="""naver-clova-ix/donut-base-finetuned-docvqa"""    ,						tokenizer=AutoTokenizer.from_pretrained("""naver-clova-ix/donut-base-finetuned-docvqa"""						)    ,						feature_extractor="""naver-clova-ix/donut-base-finetuned-docvqa"""    ,						)

       lowerCAmelCase_					:       Union[str, Any]    =		INVOICE_URL
       lowerCAmelCase_					:       int    =		"""What is the invoice number?"""
       lowerCAmelCase_					:       str    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(nested_simplify(_lowercase    ,						decimals=4						)    ,						[{"""answer""": """us-001"""}]						)







  @require_tf
  @unittest.skip("""Document question answering not implemented in TF"""						)
  def 	UpperCAmelCase__		(	self						):
       pass
