





"""simple docstring"""






import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer



class 	lowerCAmelCase					(      _UpperCamelCase							):






    def 							__A (   self     ):
           _UpperCAmelCase     =     tempfile.mkdtemp()
           _UpperCAmelCase     =     5

           # Realm tok
           _UpperCAmelCase     =     [
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''test''',
               '''question''',
               '''this''',
               '''is''',
               '''the''',
               '''first''',
               '''second''',
               '''third''',
               '''fourth''',
               '''fifth''',
               '''record''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           _UpperCAmelCase     =     os.path.join(self.tmpdirname    ,  'realm_tokenizer'     )
           os.makedirs(_UpperCAmelCase    ,  exist_ok=_UpperCAmelCase     )
           _UpperCAmelCase     =     os.path.join(_UpperCAmelCase    ,  VOCAB_FILES_NAMES['vocab_file']     )
           with open(self.vocab_file    ,  'w'    ,  encoding='utf-8'     ) as vocab_writer:
                  vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )

           _UpperCAmelCase     =     os.path.join(self.tmpdirname    ,  'realm_block_records'     )
           os.makedirs(_UpperCAmelCase    ,  exist_ok=_UpperCAmelCase     )






    def 							__A (   self     ):
           return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname    ,  'realm_tokenizer'     )     )






    def 							__A (   self     ):
           shutil.rmtree(self.tmpdirname     )






    def 							__A (   self     ):
           _UpperCAmelCase     =     RealmConfig(num_block_records=self.num_block_records     )
           return config






    def 							__A (   self     ):
           _UpperCAmelCase     =     Dataset.from_dict(
               {
                   'id': ['0', '1'],
                   'question': ['foo', 'bar'],
                   'answers': [['Foo', 'Bar'], ['Bar']],
               }     )
           return dataset






    def 							__A (   self     ):
           _UpperCAmelCase     =     np.array(
               [
                   b'This is the first record',
                   b'This is the second record',
                   b'This is the third record',
                   b'This is the fourth record',
                   b'This is the fifth record',
                   b'This is a longer longer longer record',
               ]    ,  dtype=_UpperCAmelCase    ,  )
           return block_records






    def 							__A (   self     ):
           _UpperCAmelCase     =     RealmRetriever(
               block_records=self.get_dummy_block_records()    ,  tokenizer=self.get_tokenizer()    ,  )
           return retriever






    def 							__A (   self     ):
           _UpperCAmelCase     =     self.get_config()
           _UpperCAmelCase     =     self.get_dummy_retriever()
           _UpperCAmelCase     =     retriever.tokenizer

           _UpperCAmelCase     =     np.array([0, 3]    ,  dtype='long'     )
           _UpperCAmelCase     =     tokenizer(['Test question']     ).input_ids
           _UpperCAmelCase     =     tokenizer(
               ['the fourth']    ,  add_special_tokens=_UpperCAmelCase    ,  return_token_type_ids=_UpperCAmelCase    ,  return_attention_mask=_UpperCAmelCase    ,  ).input_ids
           _UpperCAmelCase     =     config.reader_seq_len

           _UpperCAmelCase     =     retriever(
               _UpperCAmelCase    ,  _UpperCAmelCase    ,  answer_ids=_UpperCAmelCase    ,  max_length=_UpperCAmelCase    ,  return_tensors='np'     )

           self.assertEqual(len(_UpperCAmelCase     )    ,  2     )
           self.assertEqual(len(_UpperCAmelCase     )    ,  2     )
           self.assertEqual(len(_UpperCAmelCase     )    ,  2     )
           self.assertEqual(concat_inputs.input_ids.shape    ,  (2, 10)     )
           self.assertEqual(concat_inputs.attention_mask.shape    ,  (2, 10)     )
           self.assertEqual(concat_inputs.token_type_ids.shape    ,  (2, 10)     )
           self.assertEqual(concat_inputs.special_tokens_mask.shape    ,  (2, 10)     )
           self.assertEqual(
               tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]     )    ,  ['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']    ,  )
           self.assertEqual(
               tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]     )    ,  ['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']    ,  )






    def 							__A (   self     ):
           _UpperCAmelCase     =     self.get_config()
           _UpperCAmelCase     =     self.get_dummy_retriever()
           _UpperCAmelCase     =     retriever.tokenizer

           _UpperCAmelCase     =     np.array([0, 3, 5]    ,  dtype='long'     )
           _UpperCAmelCase     =     tokenizer(['Test question']     ).input_ids
           _UpperCAmelCase     =     tokenizer(
               ['the fourth', 'longer longer']    ,  add_special_tokens=_UpperCAmelCase    ,  return_token_type_ids=_UpperCAmelCase    ,  return_attention_mask=_UpperCAmelCase    ,  ).input_ids
           _UpperCAmelCase     =     config.reader_seq_len

           _UpperCAmelCase     =     retriever(
               _UpperCAmelCase    ,  _UpperCAmelCase    ,  answer_ids=_UpperCAmelCase    ,  max_length=_UpperCAmelCase    ,  return_tensors='np'     )

           self.assertEqual([False, True, True]    ,  _UpperCAmelCase     )
           self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]    ,  _UpperCAmelCase     )
           self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]    ,  _UpperCAmelCase     )






    def 							__A (   self     ):
           _UpperCAmelCase     =     self.get_dummy_retriever()
           retriever.save_pretrained(os.path.join(self.tmpdirname    ,  'realm_block_records'     )     )

           # Test local path
           _UpperCAmelCase     =     retriever.from_pretrained(os.path.join(self.tmpdirname    ,  'realm_block_records'     )     )
           self.assertEqual(retriever.block_records[0]    ,  b'This is the first record'     )

           # Test mocked remote path
           with patch('transformers.models.realm.retrieval_realm.hf_hub_download'     ) as mock_hf_hub_download:
                  _UpperCAmelCase     =     os.path.join(
                      os.path.join(self.tmpdirname    ,  'realm_block_records'     )    ,  _REALM_BLOCK_RECORDS_FILENAME     )
                  _UpperCAmelCase     =     RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa'     )

           self.assertEqual(retriever.block_records[0]    ,  b'This is the first record'     )













"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_			=							logging.get_logger(__name__)

lowerCAmelCase_			=							{
    '''microsoft/trocr-base-handwritten''': (
        '''https://huggingface.co/microsoft/trocr-base-handwritten/resolve/main/config.json'''
    ),
    # See all TrOCR models at https://huggingface.co/models?filter=trocr
}



class 	lowerCAmelCase					(      snake_case							):
       lowerCAmelCase__						 =  """trocr"""
       lowerCAmelCase__						 =  ["""past_key_values"""]
       lowerCAmelCase__						 =  {
           """num_attention_heads""": """decoder_attention_heads""",
           """hidden_size""": """d_model""",
           """num_hidden_layers""": """decoder_layers""",
       }






       def __init__(   self    ,  a__=5_02_65    ,  a__=10_24    ,  a__=12    ,  a__=16    ,  a__=40_96    ,  a__="gelu"    ,  a__=5_12    ,  a__=0.1    ,  a__=0.0    ,  a__=0.0    ,  a__=2    ,  a__=0.02    ,  a__=0.0    ,  a__=True    ,  a__=False    ,  a__=True    ,  a__=True    ,  a__=1    ,  a__=0    ,  a__=2    ,  **a__    ,  ):
              _UpperCAmelCase     =     vocab_size
              _UpperCAmelCase     =     d_model
              _UpperCAmelCase     =     decoder_layers
              _UpperCAmelCase     =     decoder_attention_heads
              _UpperCAmelCase     =     decoder_ffn_dim
              _UpperCAmelCase     =     activation_function
              _UpperCAmelCase     =     max_position_embeddings
              _UpperCAmelCase     =     dropout
              _UpperCAmelCase     =     attention_dropout
              _UpperCAmelCase     =     activation_dropout
              _UpperCAmelCase     =     init_std
              _UpperCAmelCase     =     decoder_layerdrop
              _UpperCAmelCase     =     use_cache
              _UpperCAmelCase     =     scale_embedding
              _UpperCAmelCase     =     use_learned_position_embeddings
              _UpperCAmelCase     =     layernorm_embedding

              super().__init__(
                  pad_token_id=a__    ,  bos_token_id=a__    ,  eos_token_id=a__    ,  decoder_start_token_id=a__    ,  **a__    ,  )






