


import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer






class      UpperCamelCase					(		lowercase__     ):


   '''simple docstring'''







   def 		UpperCamelCase  (     self		):
        lowercase_       :List[Any]     =       tempfile.mkdtemp()
        lowercase_       :Optional[Any]     =       5

        # Realm tok
        lowercase_       :List[str]     =       [
            '''[UNK]''',
            '''[CLS]''',
            '''[SEP]''',
            '''[PAD]''',
            '''[MASK]''',
            '''test''',
            '''question''',
            '''this''',
            '''is''',
            '''the''',
            '''first''',
            '''second''',
            '''third''',
            '''fourth''',
            '''fifth''',
            '''record''',
            '''want''',
            '''##want''',
            '''##ed''',
            '''wa''',
            '''un''',
            '''runn''',
            '''##ing''',
            ''',''',
            '''low''',
            '''lowest''',
        ]
        lowercase_       :Dict     =       os.path.join(self.tmpdirname   ,							'''realm_tokenizer'''		)
        os.makedirs(UpperCamelCase_   ,							exist_ok=UpperCamelCase_		)
        lowercase_       :List[Any]     =       os.path.join(UpperCamelCase_   ,							VOCAB_FILES_NAMES['''vocab_file''']		)
        with open(self.vocab_file   ,							'''w'''   ,							encoding='''utf-8'''		) as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)

        lowercase_       :Tuple     =       os.path.join(self.tmpdirname   ,							'''realm_block_records'''		)
        os.makedirs(UpperCamelCase_   ,							exist_ok=UpperCamelCase_		)
   def 		UpperCamelCase  (     self		):
        return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,							'''realm_tokenizer'''		)		)
   def 		UpperCamelCase  (     self		):
        shutil.rmtree(self.tmpdirname		)
   def 		UpperCamelCase  (     self		):
        lowercase_       :str     =       RealmConfig(num_block_records=self.num_block_records		)
        return config
   def 		UpperCamelCase  (     self		):
        lowercase_       :int     =       Dataset.from_dict(
            {
                '''id''': ['''0''', '''1'''],
                '''question''': ['''foo''', '''bar'''],
                '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
            }		)
        return dataset
   def 		UpperCamelCase  (     self		):
        lowercase_       :Optional[int]     =       np.array(
            [
                B'''This is the first record''',
                B'''This is the second record''',
                B'''This is the third record''',
                B'''This is the fourth record''',
                B'''This is the fifth record''',
                B'''This is a longer longer longer record''',
            ]   ,							dtype=UpperCamelCase_   ,							)
        return block_records
   def 		UpperCamelCase  (     self		):
        lowercase_       :Tuple     =       RealmRetriever(
            block_records=self.get_dummy_block_records()   ,							tokenizer=self.get_tokenizer()   ,							)
        return retriever
   def 		UpperCamelCase  (     self		):
        lowercase_       :List[Any]     =       self.get_config()
        lowercase_       :Dict     =       self.get_dummy_retriever()
        lowercase_       :Tuple     =       retriever.tokenizer

        lowercase_       :Optional[int]     =       np.array([0, 3]   ,							dtype='''long'''		)
        lowercase_       :Any     =       tokenizer(['''Test question''']		).input_ids
        lowercase_       :int     =       tokenizer(
            ['''the fourth''']   ,							add_special_tokens=UpperCamelCase_   ,							return_token_type_ids=UpperCamelCase_   ,							return_attention_mask=UpperCamelCase_   ,							).input_ids
        lowercase_       :Optional[Any]     =       config.reader_seq_len

        lowercase_       :List[str]     =       retriever(
            UpperCamelCase_   ,							UpperCamelCase_   ,							answer_ids=UpperCamelCase_   ,							max_length=UpperCamelCase_   ,							return_tensors='''np'''		)

        self.assertEqual(len(UpperCamelCase_		)   ,							2		)
        self.assertEqual(len(UpperCamelCase_		)   ,							2		)
        self.assertEqual(len(UpperCamelCase_		)   ,							2		)
        self.assertEqual(concat_inputs.input_ids.shape   ,							(2, 10)		)
        self.assertEqual(concat_inputs.attention_mask.shape   ,							(2, 10)		)
        self.assertEqual(concat_inputs.token_type_ids.shape   ,							(2, 10)		)
        self.assertEqual(concat_inputs.special_tokens_mask.shape   ,							(2, 10)		)
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]		)   ,							['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']   ,							)
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]		)   ,							['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']   ,							)
   def 		UpperCamelCase  (     self		):
        lowercase_       :List[str]     =       self.get_config()
        lowercase_       :Optional[int]     =       self.get_dummy_retriever()
        lowercase_       :Tuple     =       retriever.tokenizer

        lowercase_       :List[str]     =       np.array([0, 3, 5]   ,							dtype='''long'''		)
        lowercase_       :int     =       tokenizer(['''Test question''']		).input_ids
        lowercase_       :Optional[Any]     =       tokenizer(
            ['''the fourth''', '''longer longer''']   ,							add_special_tokens=UpperCamelCase_   ,							return_token_type_ids=UpperCamelCase_   ,							return_attention_mask=UpperCamelCase_   ,							).input_ids
        lowercase_       :Dict     =       config.reader_seq_len

        lowercase_       :List[Any]     =       retriever(
            UpperCamelCase_   ,							UpperCamelCase_   ,							answer_ids=UpperCamelCase_   ,							max_length=UpperCamelCase_   ,							return_tensors='''np'''		)

        self.assertEqual([False, True, True]   ,							UpperCamelCase_		)
        self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]   ,							UpperCamelCase_		)
        self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]   ,							UpperCamelCase_		)




   def 		UpperCamelCase  (     self		):
        lowercase_       :Tuple     =       self.get_dummy_retriever()
        retriever.save_pretrained(os.path.join(self.tmpdirname   ,							'''realm_block_records'''		)		)

        # Test local path
        lowercase_       :List[str]     =       retriever.from_pretrained(os.path.join(self.tmpdirname   ,							'''realm_block_records'''		)		)
        self.assertEqual(retriever.block_records[0]   ,							B'''This is the first record'''		)

        # Test mocked remote path
        with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''		) as mock_hf_hub_download:
             lowercase_       :int     =       os.path.join(
                 os.path.join(self.tmpdirname   ,							'''realm_block_records'''		)   ,							_REALM_BLOCK_RECORDS_FILENAME		)
             lowercase_       :Dict     =       RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''		)

        self.assertEqual(retriever.block_records[0]   ,							B'''This is the first record'''		)





import argparse

import torch

from transformers import OpenAIGPTConfig, OpenAIGPTModel, load_tf_weights_in_openai_gpt
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()




def     UpperCamelCase    (      _a       ,		_a       ,		_a		)					->							List[str]:
     '''simple docstring'''





     if openai_config_file == "":
          lowercase_       :str     =       OpenAIGPTConfig()
     else:
          lowercase_       :int     =       OpenAIGPTConfig.from_json_file(_a		)
     lowercase_       :int     =       OpenAIGPTModel(_a		)

     # Load weights from numpy
     load_tf_weights_in_openai_gpt(_a       ,		_a       ,		_a		)

     # Save pytorch-model
     lowercase_       :Optional[int]     =       pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
     lowercase_       :List[str]     =       pytorch_dump_folder_path + '''/''' + CONFIG_NAME
     print(f"Save PyTorch model to {pytorch_weights_dump_path}"		)
     torch.save(model.state_dict()       ,		_a		)
     print(f"Save configuration file to {pytorch_config_dump_path}"		)
     with open(_a       ,		'''w'''       ,		encoding='''utf-8'''		) as f:
          f.write(config.to_json_string()		)


if __name__ == "__main__":
   SCREAMING_SNAKE_CASE       :		List[str]         =				argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       "--openai_checkpoint_folder_path",
       default=None,
       type=str,
       required=True,
       help="Path to the TensorFlow checkpoint path.",
   )
   parser.add_argument(
       "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
   )
   parser.add_argument(
       "--openai_config_file",
       default="",
       type=str,
       help=(
           "An optional config json file corresponding to the pre-trained OpenAI model. \n"
           "This specifies the model architecture."
       ),
   )
   SCREAMING_SNAKE_CASE       :		Optional[Any]         =				parser.parse_args()
   convert_openai_checkpoint_to_pytorch(
       args.openai_checkpoint_folder_path, args.openai_config_file, args.pytorch_dump_folder_path
   )

