





import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from ...utils import logging
from ..auto import CONFIG_MAPPING


UpperCAmelCase_					:   List[Any] 							=						logging.get_logger(__name__)

UpperCAmelCase_					:   Dict 							=						{
    """salesforce/blip2-opt-2.7b""": """https://huggingface.co/salesforce/blip2-opt-2.7b/resolve/main/config.json""",
}





class      lowercase__    (							__A  ):
  __UpperCamelCase	     =  """blip_2_vision_model"""







  def __init__(	self    ,						_lowercase=1_408    ,						_lowercase=6_144    ,						_lowercase=39    ,						_lowercase=16    ,						_lowercase=224    ,						_lowercase=14    ,						_lowercase="gelu"    ,						_lowercase=0.0_0001    ,						_lowercase=0.0    ,						_lowercase=1e-10    ,						_lowercase=True    ,						**_lowercase    ,						):
       super().__init__(**_lowercase						)

       lowerCAmelCase_					:       List[str]    =		hidden_size
       lowerCAmelCase_					:       Optional[Any]    =		intermediate_size
       lowerCAmelCase_					:       List[Any]    =		num_hidden_layers
       lowerCAmelCase_					:       Optional[int]    =		num_attention_heads
       lowerCAmelCase_					:       Any    =		patch_size
       lowerCAmelCase_					:       Any    =		image_size
       lowerCAmelCase_					:       Union[str, Any]    =		initializer_range
       lowerCAmelCase_					:       int    =		attention_dropout
       lowerCAmelCase_					:       Tuple    =		layer_norm_eps
       lowerCAmelCase_					:       Any    =		hidden_act
       lowerCAmelCase_					:       List[Any]    =		qkv_bias







  @classmethod
  def 	UpperCAmelCase__		(	cls    ,						_lowercase    ,						**_lowercase						):
       cls._set_token_in_kwargs(_lowercase						)

       lowerCAmelCase_   ,  lowerCAmelCase_					:       str    =		cls.get_config_dict(_lowercase    ,						**_lowercase						)

       # get the vision config dict if we are loading from Blip2Config
       if config_dict.get("""model_type"""						) == "blip-2":
            lowerCAmelCase_					:       Any    =		config_dict["""vision_config"""]

       if "model_type" in config_dict and hasattr(cls    ,						"""model_type"""						) and config_dict["model_type"] != cls.model_type:
            logger.warning(
                F'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
                F'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'						)

       return cls.from_dict(_lowercase    ,						**_lowercase						)





class      lowercase__    (							__A  ):
  __UpperCamelCase	     =  """blip_2_qformer"""







  def __init__(	self    ,						_lowercase=30_522    ,						_lowercase=768    ,						_lowercase=12    ,						_lowercase=12    ,						_lowercase=3_072    ,						_lowercase="gelu"    ,						_lowercase=0.1    ,						_lowercase=0.1    ,						_lowercase=512    ,						_lowercase=0.02    ,						_lowercase=1e-12    ,						_lowercase=0    ,						_lowercase="absolute"    ,						_lowercase=2    ,						_lowercase=1_408    ,						**_lowercase    ,						):
       super().__init__(pad_token_id=_lowercase    ,						**_lowercase						)

       lowerCAmelCase_					:       List[str]    =		vocab_size
       lowerCAmelCase_					:       Dict    =		hidden_size
       lowerCAmelCase_					:       int    =		num_hidden_layers
       lowerCAmelCase_					:       Union[str, Any]    =		num_attention_heads
       lowerCAmelCase_					:       List[Any]    =		hidden_act
       lowerCAmelCase_					:       str    =		intermediate_size
       lowerCAmelCase_					:       Dict    =		hidden_dropout_prob
       lowerCAmelCase_					:       int    =		attention_probs_dropout_prob
       lowerCAmelCase_					:       List[Any]    =		max_position_embeddings
       lowerCAmelCase_					:       List[str]    =		initializer_range
       lowerCAmelCase_					:       Dict    =		layer_norm_eps
       lowerCAmelCase_					:       str    =		position_embedding_type
       lowerCAmelCase_					:       List[str]    =		cross_attention_frequency
       lowerCAmelCase_					:       int    =		encoder_hidden_size







  @classmethod
  def 	UpperCAmelCase__		(	cls    ,						_lowercase    ,						**_lowercase						):
       cls._set_token_in_kwargs(_lowercase						)

       lowerCAmelCase_   ,  lowerCAmelCase_					:       Dict    =		cls.get_config_dict(_lowercase    ,						**_lowercase						)

       # get the qformer config dict if we are loading from Blip2Config
       if config_dict.get("""model_type"""						) == "blip-2":
            lowerCAmelCase_					:       Any    =		config_dict["""qformer_config"""]

       if "model_type" in config_dict and hasattr(cls    ,						"""model_type"""						) and config_dict["model_type"] != cls.model_type:
            logger.warning(
                F'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
                F'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'						)

       return cls.from_dict(_lowercase    ,						**_lowercase						)





class      lowercase__    (							__A  ):
  __UpperCamelCase	     =  """blip-2"""
  __UpperCamelCase	     =  True







  def __init__(	self    ,						_lowercase=None    ,						_lowercase=None    ,						_lowercase=None    ,						_lowercase=32    ,						**_lowercase						):
       super().__init__(**_lowercase						)

       if vision_config is None:
            lowerCAmelCase_					:       Optional[int]    =		{}
            logger.info("""vision_config is None. initializing the Blip2VisionConfig with default values."""						)

       if qformer_config is None:
            lowerCAmelCase_					:       Union[str, Any]    =		{}
            logger.info("""qformer_config is None. Initializing the Blip2QFormerConfig with default values."""						)

       if text_config is None:
            lowerCAmelCase_					:       List[Any]    =		{}
            logger.info("""text_config is None. Initializing the text config with default values (`OPTConfig`)."""						)

       lowerCAmelCase_					:       Any    =		BlipaVisionConfig(**_lowercase						)
       lowerCAmelCase_					:       Union[str, Any]    =		BlipaQFormerConfig(**_lowercase						)
       lowerCAmelCase_					:       Any    =		text_config["""model_type"""] if """model_type""" in text_config else """opt"""
       lowerCAmelCase_					:       str    =		CONFIG_MAPPING[text_model_type](**_lowercase						)

       lowerCAmelCase_					:       List[str]    =		self.text_config.tie_word_embeddings
       lowerCAmelCase_					:       str    =		self.text_config.is_encoder_decoder

       lowerCAmelCase_					:       Any    =		num_query_tokens
       lowerCAmelCase_					:       Optional[Any]    =		self.vision_config.hidden_size
       lowerCAmelCase_					:       Optional[Any]    =		self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
       lowerCAmelCase_					:       Dict    =		1.0
       lowerCAmelCase_					:       List[Any]    =		0.02







  @classmethod
  def 	UpperCAmelCase__		(	cls    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						**_lowercase    ,						):
       return cls(
           vision_config=vision_config.to_dict()    ,						qformer_config=qformer_config.to_dict()    ,						text_config=text_config.to_dict()    ,						**_lowercase    ,						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Any    =		copy.deepcopy(self.__dict__						)
       lowerCAmelCase_					:       Any    =		self.vision_config.to_dict()
       lowerCAmelCase_					:       Optional[Any]    =		self.qformer_config.to_dict()
       lowerCAmelCase_					:       Any    =		self.text_config.to_dict()
       lowerCAmelCase_					:       Tuple    =		self.__class__.model_type
       return output







import unittest

from transformers import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING, AutoTokenizer, is_vision_available
from transformers.pipelines import pipeline
from transformers.pipelines.document_question_answering import apply_tesseract
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_detectrona,
    require_pytesseract,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
      from PIL import Image

      from transformers.image_utils import load_image
else:





      class      lowercase__    :







        @staticmethod
        def 	UpperCAmelCase__		(	*_lowercase    ,						**_lowercase						):
             pass






      def 							_lowerCAmelCase (  _a   :					List[str]			)	->			Any:
           return None


# This is a pinned image from a specific revision of a document question answering space, hosted by HuggingFace,
# so we can expect it to be available.
UpperCAmelCase_					:   Optional[Any] 							=						(
    """https://huggingface.co/spaces/impira/docquery/resolve/2f6c96314dc84dfda62d40de9da55f2f5165d403/invoice.png"""
)





@is_pipeline_test
@require_torch
@require_vision
class      lowercase__    (							unittest.TestCase  ):
  __UpperCamelCase	     =  MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING







  @require_pytesseract
  @require_vision
  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       Optional[int]    =		pipeline(
           """document-question-answering"""    ,						model=_lowercase    ,						tokenizer=_lowercase    ,						image_processor=_lowercase						)

       lowerCAmelCase_					:       List[str]    =		INVOICE_URL
       lowerCAmelCase_					:       str    =		list(zip(*apply_tesseract(load_image(_lowercase						)    ,						_lowercase    ,						""""""						)						)						)
       lowerCAmelCase_					:       int    =		"""What is the placebo?"""
       lowerCAmelCase_					:       str    =		[
           {
               """image""": load_image(_lowercase						),
               """question""": question,
           },
           {
               """image""": image,
               """question""": question,
           },
           {
               """image""": image,
               """question""": question,
               """word_boxes""": word_boxes,
           },
       ]
       return dqa_pipeline, examples







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       List[str]    =		dqa_pipeline(_lowercase    ,						top_k=2						)
       self.assertEqual(
           _lowercase    ,						[
               [
                   {"""score""": ANY(_lowercase						), """answer""": ANY(_lowercase						), """start""": ANY(_lowercase						), """end""": ANY(_lowercase						)},
                   {"""score""": ANY(_lowercase						), """answer""": ANY(_lowercase						), """start""": ANY(_lowercase						), """end""": ANY(_lowercase						)},
               ]
           ]
           * 3    ,						)







  @require_torch
  @require_detectrona
  @require_pytesseract
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Tuple    =		pipeline("""document-question-answering"""    ,						model="""hf-internal-testing/tiny-random-layoutlmv2"""						)
       lowerCAmelCase_					:       str    =		INVOICE_URL
       lowerCAmelCase_					:       int    =		"""How many cats are there?"""

       lowerCAmelCase_					:       List[Any]    =		[
           {"""score""": 0.0001, """answer""": """oy 2312/2019""", """start""": 38, """end""": 39},
           {"""score""": 0.0001, """answer""": """oy 2312/2019 DUE""", """start""": 38, """end""": 40},
       ]
       lowerCAmelCase_					:       Optional[Any]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(nested_simplify(_lowercase    ,						decimals=4						)    ,						_lowercase						)

       lowerCAmelCase_					:       Any    =		dqa_pipeline({"""image""": image, """question""": question}    ,						top_k=2						)
       self.assertEqual(nested_simplify(_lowercase    ,						decimals=4						)    ,						_lowercase						)

       # This image does not detect ANY text in it, meaning layoutlmv2 should fail.
       # Empty answer probably
       lowerCAmelCase_					:       List[Any]    =		"""./tests/fixtures/tests_samples/COCO/000000039769.png"""
       lowerCAmelCase_					:       Any    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(_lowercase    ,						[]						)

       # We can optionnally pass directly the words and bounding boxes
       lowerCAmelCase_					:       int    =		"""./tests/fixtures/tests_samples/COCO/000000039769.png"""
       lowerCAmelCase_					:       int    =		[]
       lowerCAmelCase_					:       List[Any]    =		[]
       lowerCAmelCase_					:       List[Any]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						words=_lowercase    ,						boxes=_lowercase    ,						top_k=2						)
       self.assertEqual(_lowercase    ,						[]						)







  @slow
  @require_torch
  @require_detectrona
  @require_pytesseract
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[Any]    =		pipeline(
           """document-question-answering"""    ,						model="""tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa"""    ,						revision="""9977165"""    ,						)
       lowerCAmelCase_					:       Dict    =		INVOICE_URL
       lowerCAmelCase_					:       int    =		"""What is the invoice number?"""

       lowerCAmelCase_					:       Union[str, Any]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9944, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.0009, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)

       lowerCAmelCase_					:       List[Any]    =		dqa_pipeline({"""image""": image, """question""": question}    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9944, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.0009, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)

       lowerCAmelCase_					:       List[str]    =		dqa_pipeline(
           [{"""image""": image, """question""": question}, {"""image""": image, """question""": question}]    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               [
                   {"""score""": 0.9944, """answer""": """us-001""", """start""": 16, """end""": 16},
                   {"""score""": 0.0009, """answer""": """us-001""", """start""": 16, """end""": 16},
               ],
           ]
           * 2    ,						)







  @slow
  @require_torch
  @require_detectrona
  @require_pytesseract
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[int]    =		pipeline(
           """document-question-answering"""    ,						model="""tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa"""    ,						revision="""9977165"""    ,						max_seq_len=50    ,						)
       lowerCAmelCase_					:       List[str]    =		INVOICE_URL
       lowerCAmelCase_					:       Optional[int]    =		"""What is the invoice number?"""

       lowerCAmelCase_					:       List[Any]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9974, """answer""": """1110212019""", """start""": 23, """end""": 23},
               {"""score""": 0.9948, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)

       lowerCAmelCase_					:       Optional[Any]    =		dqa_pipeline({"""image""": image, """question""": question}    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9974, """answer""": """1110212019""", """start""": 23, """end""": 23},
               {"""score""": 0.9948, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)

       lowerCAmelCase_					:       List[Any]    =		dqa_pipeline(
           [{"""image""": image, """question""": question}, {"""image""": image, """question""": question}]    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               [
                   {"""score""": 0.9974, """answer""": """1110212019""", """start""": 23, """end""": 23},
                   {"""score""": 0.9948, """answer""": """us-001""", """start""": 16, """end""": 16},
               ]
           ]
           * 2    ,						)







  @slow
  @require_torch
  @require_pytesseract
  @require_vision
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       int    =		AutoTokenizer.from_pretrained(
           """impira/layoutlm-document-qa"""    ,						revision="""3dc6de3"""    ,						add_prefix_space=_lowercase						)
       lowerCAmelCase_					:       Any    =		pipeline(
           """document-question-answering"""    ,						model="""impira/layoutlm-document-qa"""    ,						tokenizer=_lowercase    ,						revision="""3dc6de3"""    ,						)
       lowerCAmelCase_					:       int    =		INVOICE_URL
       lowerCAmelCase_					:       Union[str, Any]    =		"""What is the invoice number?"""

       lowerCAmelCase_					:       Optional[Any]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.4251, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.0819, """answer""": """1110212019""", """start""": 23, """end""": 23},
           ]    ,						)

       lowerCAmelCase_					:       int    =		dqa_pipeline({"""image""": image, """question""": question}    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.4251, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.0819, """answer""": """1110212019""", """start""": 23, """end""": 23},
           ]    ,						)

       lowerCAmelCase_					:       Union[str, Any]    =		dqa_pipeline(
           [{"""image""": image, """question""": question}, {"""image""": image, """question""": question}]    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               [
                   {"""score""": 0.4251, """answer""": """us-001""", """start""": 16, """end""": 16},
                   {"""score""": 0.0819, """answer""": """1110212019""", """start""": 23, """end""": 23},
               ]
           ]
           * 2    ,						)

       lowerCAmelCase_					:       int    =		list(zip(*apply_tesseract(load_image(_lowercase						)    ,						_lowercase    ,						""""""						)						)						)

       # This model should also work if `image` is set to None
       lowerCAmelCase_					:       List[Any]    =		dqa_pipeline({"""image""": None, """word_boxes""": word_boxes, """question""": question}    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.4251, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.0819, """answer""": """1110212019""", """start""": 23, """end""": 23},
           ]    ,						)







  @slow
  @require_torch
  @require_pytesseract
  @require_vision
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       int    =		AutoTokenizer.from_pretrained(
           """impira/layoutlm-document-qa"""    ,						revision="""3dc6de3"""    ,						add_prefix_space=_lowercase						)
       lowerCAmelCase_					:       Optional[Any]    =		pipeline(
           """document-question-answering"""    ,						model="""impira/layoutlm-document-qa"""    ,						tokenizer=_lowercase    ,						revision="""3dc6de3"""    ,						max_seq_len=50    ,						)
       lowerCAmelCase_					:       Tuple    =		INVOICE_URL
       lowerCAmelCase_					:       Tuple    =		"""What is the invoice number?"""

       lowerCAmelCase_					:       Optional[int]    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9999, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.9998, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)

       lowerCAmelCase_					:       int    =		dqa_pipeline(
           [{"""image""": image, """question""": question}, {"""image""": image, """question""": question}]    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               [
                   {"""score""": 0.9999, """answer""": """us-001""", """start""": 16, """end""": 16},
                   {"""score""": 0.9998, """answer""": """us-001""", """start""": 16, """end""": 16},
               ]
           ]
           * 2    ,						)

       lowerCAmelCase_					:       Union[str, Any]    =		list(zip(*apply_tesseract(load_image(_lowercase						)    ,						_lowercase    ,						""""""						)						)						)

       # This model should also work if `image` is set to None
       lowerCAmelCase_					:       List[str]    =		dqa_pipeline({"""image""": None, """word_boxes""": word_boxes, """question""": question}    ,						top_k=2						)
       self.assertEqual(
           nested_simplify(_lowercase    ,						decimals=4						)    ,						[
               {"""score""": 0.9999, """answer""": """us-001""", """start""": 16, """end""": 16},
               {"""score""": 0.9998, """answer""": """us-001""", """start""": 16, """end""": 16},
           ]    ,						)







  @slow
  @require_torch
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       int    =		pipeline(
           """document-question-answering"""    ,						model="""naver-clova-ix/donut-base-finetuned-docvqa"""    ,						tokenizer=AutoTokenizer.from_pretrained("""naver-clova-ix/donut-base-finetuned-docvqa"""						)    ,						feature_extractor="""naver-clova-ix/donut-base-finetuned-docvqa"""    ,						)

       lowerCAmelCase_					:       Union[str, Any]    =		INVOICE_URL
       lowerCAmelCase_					:       int    =		"""What is the invoice number?"""
       lowerCAmelCase_					:       str    =		dqa_pipeline(image=_lowercase    ,						question=_lowercase    ,						top_k=2						)
       self.assertEqual(nested_simplify(_lowercase    ,						decimals=4						)    ,						[{"""answer""": """us-001"""}]						)







  @require_tf
  @unittest.skip("""Document question answering not implemented in TF"""						)
  def 	UpperCAmelCase__		(	self						):
       pass
