


'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import numpy
     import tensorflow as tf

     from transformers import (
         TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
         TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
         TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
         BertConfig,
         DPRConfig,
         TFDPRContextEncoder,
         TFDPRQuestionEncoder,
         TFDPRReader,
     )





class   snake_case					:





  """simple docstring"""




  def __init__(					self    ,						UpperCamelCase    ,						UpperCamelCase=13    ,						UpperCamelCase=7    ,						UpperCamelCase=True    ,						UpperCamelCase=True    ,						UpperCamelCase=True    ,						UpperCamelCase=True    ,						UpperCamelCase=99    ,						UpperCamelCase=32    ,						UpperCamelCase=2    ,						UpperCamelCase=4    ,						UpperCamelCase=37    ,						UpperCamelCase="gelu"    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.1    ,						UpperCamelCase=512    ,						UpperCamelCase=16    ,						UpperCamelCase=2    ,						UpperCamelCase=0.02    ,						UpperCamelCase=3    ,						UpperCamelCase=4    ,						UpperCamelCase=None    ,						UpperCamelCase=0    ,						):







   """simple docstring"""
   lowerCamelCase_  				=						parent
   lowerCamelCase_  				=						batch_size
   lowerCamelCase_  				=						seq_length
   lowerCamelCase_  				=						is_training
   lowerCamelCase_  				=						use_input_mask
   lowerCamelCase_  				=						use_token_type_ids
   lowerCamelCase_  				=						use_labels
   lowerCamelCase_  				=						vocab_size
   lowerCamelCase_  				=						hidden_size
   lowerCamelCase_  				=						num_hidden_layers
   lowerCamelCase_  				=						num_attention_heads
   lowerCamelCase_  				=						intermediate_size
   lowerCamelCase_  				=						hidden_act
   lowerCamelCase_  				=						hidden_dropout_prob
   lowerCamelCase_  				=						attention_probs_dropout_prob
   lowerCamelCase_  				=						max_position_embeddings
   lowerCamelCase_  				=						type_vocab_size
   lowerCamelCase_  				=						type_sequence_label_size
   lowerCamelCase_  				=						initializer_range
   lowerCamelCase_  				=						num_labels
   lowerCamelCase_  				=						num_choices
   lowerCamelCase_  				=						scope
   lowerCamelCase_  				=						projection_dim






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						ids_tensor([self.batch_size, self.seq_length]    ,						self.vocab_size  )

   lowerCamelCase_  				=						None
   if self.use_input_mask:
    # follow test_modeling_tf_ctrl.py
    lowerCamelCase_  				=						random_attention_mask([self.batch_size, self.seq_length]  )

   lowerCamelCase_  				=						None
   if self.use_token_type_ids:
    lowerCamelCase_  				=						ids_tensor([self.batch_size, self.seq_length]    ,						self.type_vocab_size  )

   lowerCamelCase_  				=						None
   lowerCamelCase_  				=						None
   lowerCamelCase_  				=						None
   if self.use_labels:
    lowerCamelCase_  				=						ids_tensor([self.batch_size]    ,						self.type_sequence_label_size  )
    lowerCamelCase_  				=						ids_tensor([self.batch_size, self.seq_length]    ,						self.num_labels  )
    lowerCamelCase_  				=						ids_tensor([self.batch_size]    ,						self.num_choices  )

   lowerCamelCase_  				=						BertConfig(
       vocab_size=self.vocab_size    ,						hidden_size=self.hidden_size    ,						num_hidden_layers=self.num_hidden_layers    ,						num_attention_heads=self.num_attention_heads    ,						intermediate_size=self.intermediate_size    ,						hidden_act=self.hidden_act    ,						hidden_dropout_prob=self.hidden_dropout_prob    ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,						max_position_embeddings=self.max_position_embeddings    ,						type_vocab_size=self.type_vocab_size    ,						is_decoder=UpperCamelCase    ,						initializer_range=self.initializer_range    ,						)
   lowerCamelCase_  				=						DPRConfig(projection_dim=self.projection_dim    ,						**config.to_dict()  )

   return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						TFDPRContextEncoder(config=UpperCamelCase  )
   lowerCamelCase_  				=						model(UpperCamelCase    ,						attention_mask=UpperCamelCase    ,						token_type_ids=UpperCamelCase  )
   lowerCamelCase_  				=						model(UpperCamelCase    ,						token_type_ids=UpperCamelCase  )
   lowerCamelCase_  				=						model(UpperCamelCase  )
   self.parent.assertEqual(result.pooler_output.shape    ,						(self.batch_size, self.projection_dim or self.hidden_size)  )






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						TFDPRQuestionEncoder(config=UpperCamelCase  )
   lowerCamelCase_  				=						model(UpperCamelCase    ,						attention_mask=UpperCamelCase    ,						token_type_ids=UpperCamelCase  )
   lowerCamelCase_  				=						model(UpperCamelCase    ,						token_type_ids=UpperCamelCase  )
   lowerCamelCase_  				=						model(UpperCamelCase  )
   self.parent.assertEqual(result.pooler_output.shape    ,						(self.batch_size, self.projection_dim or self.hidden_size)  )






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						TFDPRReader(config=UpperCamelCase  )
   lowerCamelCase_  				=						model(UpperCamelCase    ,						attention_mask=UpperCamelCase  )

   self.parent.assertEqual(result.start_logits.shape    ,						(self.batch_size, self.seq_length)  )
   self.parent.assertEqual(result.end_logits.shape    ,						(self.batch_size, self.seq_length)  )
   self.parent.assertEqual(result.relevance_logits.shape    ,						(self.batch_size,)  )







  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.prepare_config_and_inputs()
   (
       (
       lowerCamelCase_
   )				,(
       lowerCamelCase_
   )				,(
       lowerCamelCase_
   )				,(
       lowerCamelCase_
   )				,(
       lowerCamelCase_
   )				,(
       lowerCamelCase_
   )				,(
       lowerCamelCase_
   )				,
   )  				=						config_and_inputs
   lowerCamelCase_  				=						{"input_ids": input_ids}
   return config, inputs_dict





@require_tf
class   snake_case					(					lowercase    ,		lowercase    ,		unittest.TestCase    ):





  """simple docstring"""




  _lowerCamelCase						=						(
      (
          TFDPRContextEncoder,
          TFDPRQuestionEncoder,
          TFDPRReader,
      )
      if is_tf_available()
      else ()
  )
  _lowerCamelCase						=						{"feature-extraction": TFDPRQuestionEncoder} if is_tf_available() else {}

  _lowerCamelCase						=						False
  _lowerCamelCase						=						False
  _lowerCamelCase						=						False
  _lowerCamelCase						=						False
  _lowerCamelCase						=						False






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						TFDPRModelTester(self  )
   lowerCamelCase_  				=						ConfigTester(self    ,						config_class=UpperCamelCase    ,						hidden_size=37  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   self.config_tester.run_common_tests()






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_dpr_context_encoder(*UpperCamelCase  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_dpr_question_encoder(*UpperCamelCase  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_dpr_reader(*UpperCamelCase  )







  @slow
  def 						snake_case	(					self  ):







   """simple docstring"""
   for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
    lowerCamelCase_  				=						TFDPRContextEncoder.from_pretrained(UpperCamelCase  )
    self.assertIsNotNone(UpperCamelCase  )

   for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
    lowerCamelCase_  				=						TFDPRContextEncoder.from_pretrained(UpperCamelCase  )
    self.assertIsNotNone(UpperCamelCase  )

   for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
    lowerCamelCase_  				=						TFDPRQuestionEncoder.from_pretrained(UpperCamelCase  )
    self.assertIsNotNone(UpperCamelCase  )

   for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
    lowerCamelCase_  				=						TFDPRReader.from_pretrained(UpperCamelCase  )
    self.assertIsNotNone(UpperCamelCase  )





@require_tf
class   snake_case					(					unittest.TestCase    ):





  """simple docstring"""




  @slow
  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						TFDPRQuestionEncoder.from_pretrained("facebook/dpr-question_encoder-single-nq-base"  )

   lowerCamelCase_  				=						tf.constant(
       [[101, 7592, 1010, 2003, 2026, 3899, 1_0140, 1029, 102]]  )  # [CLS] hello, is my dog cute? [SEP]
   lowerCamelCase_  				=						model(UpperCamelCase  )[0]  # embedding shape = (1, 768)
   # compare the actual values for a slice.
   lowerCamelCase_  				=						tf.constant(
       [
           [
               0.03_236_253,
               0.12_753_335,
               0.16_818_509,
               0.00_279_786,
               0.3_896_933,
               0.24_264_945,
               0.2_178_971,
               -0.02_335_227,
               -0.08_481_959,
               -0.14_324_117,
           ]
       ]  )
   self.assertTrue(numpy.allclose(output[:, :10].numpy()    ,						expected_slice.numpy()    ,						atol=1e-4  )  )







'''simple docstring'''

import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


a_						:       Dict      			=				logging.get_logger(__name__)

a_						:       List[Any]      			=				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

a_						:       Optional[Any]      			=				{
    """tokenizer_file""": {
        """EleutherAI/gpt-neox-20b""": """https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json""",
    },
}

a_						:       str      			=				{
    """gpt-neox-20b""": 2048,
}





class   snake_case					(					lowercase    ):





  """simple docstring"""




  _lowerCamelCase						=						VOCAB_FILES_NAMES
  _lowerCamelCase						=						PRETRAINED_VOCAB_FILES_MAP
  _lowerCamelCase						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _lowerCamelCase						=						["input_ids", "attention_mask"]






  def __init__(					self    ,						UpperCamelCase=None    ,						UpperCamelCase=None    ,						UpperCamelCase=None    ,						UpperCamelCase="<|endoftext|>"    ,						UpperCamelCase="<|endoftext|>"    ,						UpperCamelCase="<|endoftext|>"    ,						UpperCamelCase=False    ,						**UpperCamelCase    ,						):







   """simple docstring"""
   super().__init__(
       UpperCamelCase    ,						UpperCamelCase    ,						tokenizer_file=UpperCamelCase    ,						unk_token=UpperCamelCase    ,						bos_token=UpperCamelCase    ,						eos_token=UpperCamelCase    ,						add_prefix_space=UpperCamelCase    ,						**UpperCamelCase    ,						)

   lowerCamelCase_  				=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()  )
   if pre_tok_state.get("add_prefix_space"    ,						UpperCamelCase  ) != add_prefix_space:
    lowerCamelCase_  				=						getattr(UpperCamelCase    ,						pre_tok_state.pop("type"  )  )
    lowerCamelCase_  				=						add_prefix_space
    lowerCamelCase_  				=						pre_tok_class(**UpperCamelCase  )

   lowerCamelCase_  				=						add_prefix_space






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase = None  ):







   """simple docstring"""
   lowerCamelCase_  				=						self._tokenizer.model.save(UpperCamelCase    ,						name=UpperCamelCase  )
   return tuple(UpperCamelCase  )







  def 						snake_case	(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						[]
   for is_user, text in conversation.iter_texts():
    input_ids.extend(self.encode(UpperCamelCase    ,						add_special_tokens=UpperCamelCase  ) + [self.eos_token_id]  )

   if len(UpperCamelCase  ) > self.model_max_length:
    lowerCamelCase_  				=						input_ids[-self.model_max_length :]
   return input_ids



