





'''simple docstring'''

import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer







class __a (   _snake_case			):





   def      UpperCAmelCase__     (					self				:			str    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       tempfile.mkdtemp()
      __SCREAMING_SNAKE_CASE      							=       5

      # Realm tok
      __SCREAMING_SNAKE_CASE      							=       [
          """[UNK]""",
          """[CLS]""",
          """[SEP]""",
          """[PAD]""",
          """[MASK]""",
          """test""",
          """question""",
          """this""",
          """is""",
          """the""",
          """first""",
          """second""",
          """third""",
          """fourth""",
          """fifth""",
          """record""",
          """want""",
          """##want""",
          """##ed""",
          """wa""",
          """un""",
          """runn""",
          """##ing""",
          """,""",
          """low""",
          """lowest""",
      ]
      __SCREAMING_SNAKE_CASE      							=       os.path.join(self.tmpdirname				,"""realm_tokenizer"""    )
      os.makedirs(lowerCamelCase				,exist_ok=lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       os.path.join(lowerCamelCase				,VOCAB_FILES_NAMES["""vocab_file"""]    )
      with open(self.vocab_file				,"""w"""				,encoding="""utf-8"""    ) as vocab_writer:
         vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )

      __SCREAMING_SNAKE_CASE      							=       os.path.join(self.tmpdirname				,"""realm_block_records"""    )
      os.makedirs(lowerCamelCase				,exist_ok=lowerCamelCase    )





   def      UpperCAmelCase__     (					self				:			Any    ):

      '''simple docstring'''

      return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname				,"""realm_tokenizer"""    )    )





   def      UpperCAmelCase__     (					self				:			int    ):

      '''simple docstring'''

      shutil.rmtree(self.tmpdirname    )





   def      UpperCAmelCase__     (					self				:			Optional[Any]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       RealmConfig(num_block_records=self.num_block_records    )
      return config





   def      UpperCAmelCase__     (					self				:			Any    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       Dataset.from_dict(
          {
              """id""": ["""0""", """1"""],
              """question""": ["""foo""", """bar"""],
              """answers""": [["""Foo""", """Bar"""], ["""Bar"""]],
          }    )
      return dataset





   def      UpperCAmelCase__     (					self				:			Optional[int]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       np.array(
          [
              b"""This is the first record""",
              b"""This is the second record""",
              b"""This is the third record""",
              b"""This is the fourth record""",
              b"""This is the fifth record""",
              b"""This is a longer longer longer record""",
          ]				,dtype=lowerCamelCase				,)
      return block_records





   def      UpperCAmelCase__     (					self				:			List[Any]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       RealmRetriever(
          block_records=self.get_dummy_block_records()				,tokenizer=self.get_tokenizer()				,)
      return retriever





   def      UpperCAmelCase__     (					self				:			List[Any]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       self.get_config()
      __SCREAMING_SNAKE_CASE      							=       self.get_dummy_retriever()
      __SCREAMING_SNAKE_CASE      							=       retriever.tokenizer

      __SCREAMING_SNAKE_CASE      							=       np.array([0, 3]				,dtype="""long"""    )
      __SCREAMING_SNAKE_CASE      							=       tokenizer(["""Test question"""]    ).input_ids
      __SCREAMING_SNAKE_CASE      							=       tokenizer(
          ["""the fourth"""]				,add_special_tokens=lowerCamelCase				,return_token_type_ids=lowerCamelCase				,return_attention_mask=lowerCamelCase				,).input_ids
      __SCREAMING_SNAKE_CASE      							=       config.reader_seq_len

      __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       retriever(
          lowerCamelCase				,lowerCamelCase				,answer_ids=lowerCamelCase				,max_length=lowerCamelCase				,return_tensors="""np"""    )

      self.assertEqual(len(lowerCamelCase    )				,2    )
      self.assertEqual(len(lowerCamelCase    )				,2    )
      self.assertEqual(len(lowerCamelCase    )				,2    )
      self.assertEqual(concat_inputs.input_ids.shape				,(2, 10)    )
      self.assertEqual(concat_inputs.attention_mask.shape				,(2, 10)    )
      self.assertEqual(concat_inputs.token_type_ids.shape				,(2, 10)    )
      self.assertEqual(concat_inputs.special_tokens_mask.shape				,(2, 10)    )
      self.assertEqual(
          tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]    )				,["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """first""", """record""", """[SEP]"""]				,)
      self.assertEqual(
          tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]    )				,["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """fourth""", """record""", """[SEP]"""]				,)





   def      UpperCAmelCase__     (					self				:			List[str]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       self.get_config()
      __SCREAMING_SNAKE_CASE      							=       self.get_dummy_retriever()
      __SCREAMING_SNAKE_CASE      							=       retriever.tokenizer

      __SCREAMING_SNAKE_CASE      							=       np.array([0, 3, 5]				,dtype="""long"""    )
      __SCREAMING_SNAKE_CASE      							=       tokenizer(["""Test question"""]    ).input_ids
      __SCREAMING_SNAKE_CASE      							=       tokenizer(
          ["""the fourth""", """longer longer"""]				,add_special_tokens=lowerCamelCase				,return_token_type_ids=lowerCamelCase				,return_attention_mask=lowerCamelCase				,).input_ids
      __SCREAMING_SNAKE_CASE      							=       config.reader_seq_len

      __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       retriever(
          lowerCamelCase				,lowerCamelCase				,answer_ids=lowerCamelCase				,max_length=lowerCamelCase				,return_tensors="""np"""    )

      self.assertEqual([False, True, True]				,lowerCamelCase    )
      self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]				,lowerCamelCase    )
      self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]				,lowerCamelCase    )





   def      UpperCAmelCase__     (					self				:			int    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       self.get_dummy_retriever()
      retriever.save_pretrained(os.path.join(self.tmpdirname				,"""realm_block_records"""    )    )

      # Test local path
      __SCREAMING_SNAKE_CASE      							=       retriever.from_pretrained(os.path.join(self.tmpdirname				,"""realm_block_records"""    )    )
      self.assertEqual(retriever.block_records[0]				,b"""This is the first record"""    )

      # Test mocked remote path
      with patch("""transformers.models.realm.retrieval_realm.hf_hub_download"""    ) as mock_hf_hub_download:
         __SCREAMING_SNAKE_CASE      							=       os.path.join(
             os.path.join(self.tmpdirname				,"""realm_block_records"""    )				,_REALM_BLOCK_RECORDS_FILENAME    )
         __SCREAMING_SNAKE_CASE      							=       RealmRetriever.from_pretrained("""google/realm-cc-news-pretrained-openqa"""    )

      self.assertEqual(retriever.block_records[0]				,b"""This is the first record"""    )









'''simple docstring'''

def 						__magic_name__       (							__UpperCAmelCase      )						->					bool:

   '''simple docstring'''







   if num < 0:
      return False

   __SCREAMING_SNAKE_CASE      							=       num
   __SCREAMING_SNAKE_CASE      							=       0
   while num > 0:
      __SCREAMING_SNAKE_CASE      							=       rev_num * 10 + (num % 10)
      num //= 10

   return num_copy == rev_num


if __name__ == "__main__":
    import doctest

    doctest.testmod()





