






'''simple docstring'''



import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer
class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      def 				A__      ( self			:		Any		)  ->  str:


       '''simple docstring'''

       lowercase					:   Tuple				  =tempfile.mkdtemp()
       lowercase					:   str				  =5

       # Realm tok
       lowercase					:   Optional[int]				  =[
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''test''',
           '''question''',
           '''this''',
           '''is''',
           '''the''',
           '''first''',
           '''second''',
           '''third''',
           '''fourth''',
           '''fifth''',
           '''record''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       lowercase					:   Optional[int]				  =os.path.join(self.tmpdirname     ,      '''realm_tokenizer'''		)
       os.makedirs(UpperCAmelCase     ,      exist_ok=UpperCAmelCase		)
       lowercase					:   Optional[Any]				  =os.path.join(UpperCAmelCase     ,      VOCAB_FILES_NAMES['''vocab_file''']		)
       with open(self.vocab_file     ,      '''w'''     ,      encoding='''utf-8'''		) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)

       lowercase					:   Dict				  =os.path.join(self.tmpdirname     ,      '''realm_block_records'''		)
       os.makedirs(UpperCAmelCase     ,      exist_ok=UpperCAmelCase		)






      def 				A__      ( self			:		Any		)  ->  RealmTokenizer:


       '''simple docstring'''

       return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname     ,      '''realm_tokenizer'''		)		)






      def 				A__      ( self			:		Dict		)  ->  Tuple:


       '''simple docstring'''

       shutil.rmtree(self.tmpdirname		)






      def 				A__      ( self			:		List[str]		)  ->  Union[str, Any]:


       '''simple docstring'''

       lowercase					:   Tuple				  =RealmConfig(num_block_records=self.num_block_records		)
       return config






      def 				A__      ( self			:		int		)  ->  Optional[Any]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =Dataset.from_dict(
           {
               '''id''': ['''0''', '''1'''],
               '''question''': ['''foo''', '''bar'''],
               '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
           }		)
       return dataset






      def 				A__      ( self			:		Tuple		)  ->  str:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =np.array(
           [
               b'''This is the first record''',
               b'''This is the second record''',
               b'''This is the third record''',
               b'''This is the fourth record''',
               b'''This is the fifth record''',
               b'''This is a longer longer longer record''',
           ]     ,      dtype=UpperCAmelCase     ,      )
       return block_records






      def 				A__      ( self			:		Optional[Any]		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =RealmRetriever(
           block_records=self.get_dummy_block_records()     ,      tokenizer=self.get_tokenizer()     ,      )
       return retriever






      def 				A__      ( self			:		Optional[int]		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   Union[str, Any]				  =self.get_config()
       lowercase					:   Tuple				  =self.get_dummy_retriever()
       lowercase					:   int				  =retriever.tokenizer

       lowercase					:   Any				  =np.array([0, 3]     ,      dtype='''long'''		)
       lowercase					:   Optional[int]				  =tokenizer(['''Test question''']		).input_ids
       lowercase					:   Optional[int]				  =tokenizer(
           ['''the fourth''']     ,      add_special_tokens=UpperCAmelCase     ,      return_token_type_ids=UpperCAmelCase     ,      return_attention_mask=UpperCAmelCase     ,      ).input_ids
       lowercase					:   List[Any]				  =config.reader_seq_len

       lowercase  ,		lowercase  ,		lowercase  ,		lowercase					:   Dict				  =retriever(
           UpperCAmelCase     ,      UpperCAmelCase     ,      answer_ids=UpperCAmelCase     ,      max_length=UpperCAmelCase     ,      return_tensors='''np'''		)

       self.assertEqual(len(UpperCAmelCase		)     ,      2		)
       self.assertEqual(len(UpperCAmelCase		)     ,      2		)
       self.assertEqual(len(UpperCAmelCase		)     ,      2		)
       self.assertEqual(concat_inputs.input_ids.shape     ,      (2, 10)		)
       self.assertEqual(concat_inputs.attention_mask.shape     ,      (2, 10)		)
       self.assertEqual(concat_inputs.token_type_ids.shape     ,      (2, 10)		)
       self.assertEqual(concat_inputs.special_tokens_mask.shape     ,      (2, 10)		)
       self.assertEqual(
           tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]		)     ,      ['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']     ,      )
       self.assertEqual(
           tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]		)     ,      ['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']     ,      )






      def 				A__      ( self			:		Tuple		)  ->  int:


       '''simple docstring'''

       lowercase					:   List[str]				  =self.get_config()
       lowercase					:   Optional[Any]				  =self.get_dummy_retriever()
       lowercase					:   int				  =retriever.tokenizer

       lowercase					:   Optional[Any]				  =np.array([0, 3, 5]     ,      dtype='''long'''		)
       lowercase					:   Union[str, Any]				  =tokenizer(['''Test question''']		).input_ids
       lowercase					:   Tuple				  =tokenizer(
           ['''the fourth''', '''longer longer''']     ,      add_special_tokens=UpperCAmelCase     ,      return_token_type_ids=UpperCAmelCase     ,      return_attention_mask=UpperCAmelCase     ,      ).input_ids
       lowercase					:   Tuple				  =config.reader_seq_len

       lowercase  ,		lowercase  ,		lowercase  ,		lowercase					:   int				  =retriever(
           UpperCAmelCase     ,      UpperCAmelCase     ,      answer_ids=UpperCAmelCase     ,      max_length=UpperCAmelCase     ,      return_tensors='''np'''		)

       self.assertEqual([False, True, True]     ,      UpperCAmelCase		)
       self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]     ,      UpperCAmelCase		)
       self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]     ,      UpperCAmelCase		)






      def 				A__      ( self			:		str		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase					:   Optional[int]				  =self.get_dummy_retriever()
       retriever.save_pretrained(os.path.join(self.tmpdirname     ,      '''realm_block_records'''		)		)

       # Test local path
       lowercase					:   List[Any]				  =retriever.from_pretrained(os.path.join(self.tmpdirname     ,      '''realm_block_records'''		)		)
       self.assertEqual(retriever.block_records[0]     ,      b'''This is the first record'''		)

       # Test mocked remote path
       with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''		) as mock_hf_hub_download:
        lowercase					:   Dict				  =os.path.join(
            os.path.join(self.tmpdirname     ,      '''realm_block_records'''		)     ,      _REALM_BLOCK_RECORDS_FILENAME		)
        lowercase					:   Dict				  =RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''		)

       self.assertEqual(retriever.block_records[0]     ,      b'''This is the first record'''		)











'''simple docstring'''



import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      UpperCamelCase_								=			['''image_processor''', '''tokenizer''']
      UpperCamelCase_								=			'''LayoutLMv2ImageProcessor'''
      UpperCamelCase_								=			('''LayoutXLMTokenizer''', '''LayoutXLMTokenizerFast''')






      def __init__( self			:		List[str]     ,      UpperCAmelCase			:		Tuple=None     ,      UpperCAmelCase			:		str=None     ,      **UpperCAmelCase			:		Dict		)  ->  Optional[int]:


       '''simple docstring'''

       if "feature_extractor" in kwargs:
        warnings.warn(
            '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
            ''' instead.'''     ,      UpperCAmelCase     ,      )
        lowercase					:   Any				  =kwargs.pop('''feature_extractor'''		)

       lowercase					:   Dict				  =image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
        raise ValueError('''You need to specify an `image_processor`.'''		)
       if tokenizer is None:
        raise ValueError('''You need to specify a `tokenizer`.'''		)

       super().__init__(UpperCAmelCase     ,      UpperCAmelCase		)






      def __call__( self			:		Any     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None     ,      UpperCAmelCase			:		Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None     ,      UpperCAmelCase			:		Union[List[List[int]], List[List[List[int]]]] = None     ,      UpperCAmelCase			:		Optional[Union[List[int], List[List[int]]]] = None     ,      UpperCAmelCase			:		bool = True     ,      UpperCAmelCase			:		Union[bool, str, PaddingStrategy] = False     ,      UpperCAmelCase			:		Union[bool, str, TruncationStrategy] = None     ,      UpperCAmelCase			:		Optional[int] = None     ,      UpperCAmelCase			:		int = 0     ,      UpperCAmelCase			:		Optional[int] = None     ,      UpperCAmelCase			:		Optional[bool] = None     ,      UpperCAmelCase			:		Optional[bool] = None     ,      UpperCAmelCase			:		bool = False     ,      UpperCAmelCase			:		bool = False     ,      UpperCAmelCase			:		bool = False     ,      UpperCAmelCase			:		bool = False     ,      UpperCAmelCase			:		bool = True     ,      UpperCAmelCase			:		Optional[Union[str, TensorType]] = None     ,      **UpperCAmelCase			:		Any     ,      )  ->  BatchEncoding:


       '''simple docstring'''

       if self.image_processor.apply_ocr and (boxes is not None):
        raise ValueError(
            '''You cannot provide bounding boxes '''
            '''if you initialized the image processor with apply_ocr set to True.'''		)

       if self.image_processor.apply_ocr and (word_labels is not None):
        raise ValueError(
            '''You cannot provide word labels if you initialized the image processor with apply_ocr set to True.'''		)

       if return_overflowing_tokens is True and return_offsets_mapping is False:
        raise ValueError('''You cannot return overflowing tokens without returning the offsets mapping.'''		)

       # first, apply the image processor
       lowercase					:   Tuple				  =self.image_processor(images=UpperCAmelCase     ,      return_tensors=UpperCAmelCase		)

       # second, apply the tokenizer
       if text is not None and self.image_processor.apply_ocr and text_pair is None:
        if isinstance(UpperCAmelCase     ,      UpperCAmelCase		):
         lowercase					:   Optional[Any]				  =[text]  # add batch dimension (as the image processor always adds a batch dimension)
        lowercase					:   List[str]				  =features['''words''']

       lowercase					:   Optional[Any]				  =self.tokenizer(
           text=text if text is not None else features['''words''']     ,      text_pair=text_pair if text_pair is not None else None     ,      boxes=boxes if boxes is not None else features['''boxes''']     ,      word_labels=UpperCAmelCase     ,      add_special_tokens=UpperCAmelCase     ,      padding=UpperCAmelCase     ,      truncation=UpperCAmelCase     ,      max_length=UpperCAmelCase     ,      stride=UpperCAmelCase     ,      pad_to_multiple_of=UpperCAmelCase     ,      return_token_type_ids=UpperCAmelCase     ,      return_attention_mask=UpperCAmelCase     ,      return_overflowing_tokens=UpperCAmelCase     ,      return_special_tokens_mask=UpperCAmelCase     ,      return_offsets_mapping=UpperCAmelCase     ,      return_length=UpperCAmelCase     ,      verbose=UpperCAmelCase     ,      return_tensors=UpperCAmelCase     ,      **UpperCAmelCase     ,      )

       # add pixel values
       lowercase					:   List[str]				  =features.pop('''pixel_values'''		)
       if return_overflowing_tokens is True:
        lowercase					:   str				  =self.get_overflowing_images(UpperCAmelCase     ,      encoded_inputs['''overflow_to_sample_mapping''']		)
       lowercase					:   Dict				  =images

       return encoded_inputs






      def 				A__      ( self			:		Any     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		str		)  ->  str:


       '''simple docstring'''

       lowercase					:   str				  =[]
       for sample_idx in overflow_to_sample_mapping:
        images_with_overflow.append(images[sample_idx]		)

       if len(UpperCAmelCase		) != len(UpperCAmelCase		):
        raise ValueError(
            '''Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got'''
            f' {len(UpperCAmelCase		)} and {len(UpperCAmelCase		)}'		)

       return images_with_overflow






      def 				A__      ( self			:		Union[str, Any]     ,      *UpperCAmelCase			:		Tuple     ,      **UpperCAmelCase			:		Dict		)  ->  int:


       '''simple docstring'''

       return self.tokenizer.batch_decode(*UpperCAmelCase     ,      **UpperCAmelCase		)






      def 				A__      ( self			:		Dict     ,      *UpperCAmelCase			:		Any     ,      **UpperCAmelCase			:		Dict		)  ->  str:


       '''simple docstring'''

       return self.tokenizer.decode(*UpperCAmelCase     ,      **UpperCAmelCase		)






      @property
      def 				A__      ( self			:		List[str]		)  ->  List[str]:


       '''simple docstring'''

       return ["input_ids", "bbox", "attention_mask", "image"]






      @property
      def 				A__      ( self			:		int		)  ->  List[str]:


       '''simple docstring'''

       warnings.warn(
           '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''     ,      UpperCAmelCase     ,      )
       return self.image_processor_class






      @property
      def 				A__      ( self			:		Dict		)  ->  List[str]:


       '''simple docstring'''

       warnings.warn(
           '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''     ,      UpperCAmelCase     ,      )
       return self.image_processor



