'''simple docstring'''
import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer







class lowerCamelCase							(	lowerCamelCase					):




  '''simple docstring'''



  def  lowerCAmelCase__ (      self			:       List[str]  )				->List[Any]:
        UpperCAmelCase_            =  tempfile.mkdtemp()
        UpperCAmelCase_            =  5

        # Realm tok
        UpperCAmelCase_            =  [
            '''[UNK]''',
            '''[CLS]''',
            '''[SEP]''',
            '''[PAD]''',
            '''[MASK]''',
            '''test''',
            '''question''',
            '''this''',
            '''is''',
            '''the''',
            '''first''',
            '''second''',
            '''third''',
            '''fourth''',
            '''fifth''',
            '''record''',
            '''want''',
            '''##want''',
            '''##ed''',
            '''wa''',
            '''un''',
            '''runn''',
            '''##ing''',
            ''',''',
            '''low''',
            '''lowest''',
        ]
        UpperCAmelCase_            =  os.path.join(self.tmpdirname  ,							'''realm_tokenizer'''  )
        os.makedirs(UpperCAmelCase__  ,							exist_ok=UpperCAmelCase__  )
        UpperCAmelCase_            =  os.path.join(UpperCAmelCase__  ,							VOCAB_FILES_NAMES['''vocab_file''']  )
        with open(self.vocab_file  ,							'''w'''  ,							encoding='''utf-8'''  ) as vocab_writer:
              vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )

        UpperCAmelCase_            =  os.path.join(self.tmpdirname  ,							'''realm_block_records'''  )
        os.makedirs(UpperCAmelCase__  ,							exist_ok=UpperCAmelCase__  )






  def  lowerCAmelCase__ (      self			:       Dict  )				->RealmTokenizer:
        return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,							'''realm_tokenizer'''  )  )






  def  lowerCAmelCase__ (      self			:       Optional[int]  )				->List[Any]:
        shutil.rmtree(self.tmpdirname  )






  def  lowerCAmelCase__ (      self			:       str  )				->Any:
        UpperCAmelCase_            =  RealmConfig(num_block_records=self.num_block_records  )
        return config






  def  lowerCAmelCase__ (      self			:       Dict  )				->Optional[int]:
        UpperCAmelCase_            =  Dataset.from_dict(
            {
                '''id''': ['''0''', '''1'''],
                '''question''': ['''foo''', '''bar'''],
                '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
            }  )
        return dataset






  def  lowerCAmelCase__ (      self			:       str  )				->Any:
        UpperCAmelCase_            =  np.array(
            [
                B'''This is the first record''',
                B'''This is the second record''',
                B'''This is the third record''',
                B'''This is the fourth record''',
                B'''This is the fifth record''',
                B'''This is a longer longer longer record''',
            ]  ,							dtype=UpperCAmelCase__  ,							)
        return block_records






  def  lowerCAmelCase__ (      self			:       Tuple  )				->Union[str, Any]:
        UpperCAmelCase_            =  RealmRetriever(
            block_records=self.get_dummy_block_records()  ,							tokenizer=self.get_tokenizer()  ,							)
        return retriever






  def  lowerCAmelCase__ (      self			:       Any  )				->Union[str, Any]:
        UpperCAmelCase_            =  self.get_config()
        UpperCAmelCase_            =  self.get_dummy_retriever()
        UpperCAmelCase_            =  retriever.tokenizer

        UpperCAmelCase_            =  np.array([0, 3]  ,							dtype='''long'''  )
        UpperCAmelCase_            =  tokenizer(['''Test question''']  ).input_ids
        UpperCAmelCase_            =  tokenizer(
            ['''the fourth''']  ,							add_special_tokens=UpperCAmelCase__  ,							return_token_type_ids=UpperCAmelCase__  ,							return_attention_mask=UpperCAmelCase__  ,							).input_ids
        UpperCAmelCase_            =  config.reader_seq_len

        UpperCAmelCase_      ,		UpperCAmelCase_      ,		UpperCAmelCase_      ,		UpperCAmelCase_            =  retriever(
            UpperCAmelCase__  ,							UpperCAmelCase__  ,							answer_ids=UpperCAmelCase__  ,							max_length=UpperCAmelCase__  ,							return_tensors='''np'''  )

        self.assertEqual(len(UpperCAmelCase__  )  ,							2  )
        self.assertEqual(len(UpperCAmelCase__  )  ,							2  )
        self.assertEqual(len(UpperCAmelCase__  )  ,							2  )
        self.assertEqual(concat_inputs.input_ids.shape  ,							(2, 10)  )
        self.assertEqual(concat_inputs.attention_mask.shape  ,							(2, 10)  )
        self.assertEqual(concat_inputs.token_type_ids.shape  ,							(2, 10)  )
        self.assertEqual(concat_inputs.special_tokens_mask.shape  ,							(2, 10)  )
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]  )  ,							['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']  ,							)
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]  )  ,							['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']  ,							)






  def  lowerCAmelCase__ (      self			:       List[str]  )				->List[str]:
        UpperCAmelCase_            =  self.get_config()
        UpperCAmelCase_            =  self.get_dummy_retriever()
        UpperCAmelCase_            =  retriever.tokenizer

        UpperCAmelCase_            =  np.array([0, 3, 5]  ,							dtype='''long'''  )
        UpperCAmelCase_            =  tokenizer(['''Test question''']  ).input_ids
        UpperCAmelCase_            =  tokenizer(
            ['''the fourth''', '''longer longer''']  ,							add_special_tokens=UpperCAmelCase__  ,							return_token_type_ids=UpperCAmelCase__  ,							return_attention_mask=UpperCAmelCase__  ,							).input_ids
        UpperCAmelCase_            =  config.reader_seq_len

        UpperCAmelCase_      ,		UpperCAmelCase_      ,		UpperCAmelCase_      ,		UpperCAmelCase_            =  retriever(
            UpperCAmelCase__  ,							UpperCAmelCase__  ,							answer_ids=UpperCAmelCase__  ,							max_length=UpperCAmelCase__  ,							return_tensors='''np'''  )

        self.assertEqual([False, True, True]  ,							UpperCAmelCase__  )
        self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]  ,							UpperCAmelCase__  )
        self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]  ,							UpperCAmelCase__  )






  def  lowerCAmelCase__ (      self			:       Union[str, Any]  )				->Union[str, Any]:
        UpperCAmelCase_            =  self.get_dummy_retriever()
        retriever.save_pretrained(os.path.join(self.tmpdirname  ,							'''realm_block_records'''  )  )

        # Test local path
        UpperCAmelCase_            =  retriever.from_pretrained(os.path.join(self.tmpdirname  ,							'''realm_block_records'''  )  )
        self.assertEqual(retriever.block_records[0]  ,							B'''This is the first record'''  )

        # Test mocked remote path
        with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''  ) as mock_hf_hub_download:
              UpperCAmelCase_            =  os.path.join(
                  os.path.join(self.tmpdirname  ,							'''realm_block_records'''  )  ,							_REALM_BLOCK_RECORDS_FILENAME  )
              UpperCAmelCase_            =  RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''  )

        self.assertEqual(retriever.block_records[0]  ,							B'''This is the first record'''  )






'''simple docstring'''
lowercase__  :      Union[str, Any]        =					"\n# Transformers 설치 방법\n! pip install transformers datasets\n# 마지막 릴리스 대신 소스에서 설치하려면, 위 명령을 주석으로 바꾸고 아래 명령을 해제하세요.\n# ! pip install git+https://github.com/huggingface/transformers.git\n"

lowercase__  :      str        =					[{"type": "code", "content": INSTALL_CONTENT}]
lowercase__  :      Any        =					{
    "{processor_class}": "FakeProcessorClass",
    "{model_class}": "FakeModelClass",
    "{object_class}": "FakeObjectClass",
}





