
import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel


class        lowercase      :



		def __init__(				self       ,   A_       ,   A_=13       ,   A_=7       ,   A_=True       ,   A_=True       ,   A_=False       ,   A_=True       ,   A_=99       ,   A_=32       ,   A_=5       ,   A_=4       ,   A_=37       ,   A_="gelu"       ,   A_=0.1       ,   A_=0.1       ,   A_=512       ,   A_=16       ,   A_=2       ,   A_=0.02       ,   A_=3       ,   A_=4       ,   A_=None       ,   )						->      str:
				"""simple docstring"""
				UpperCamelCase						=  parent
				UpperCamelCase						=  batch_size
				UpperCamelCase						=  seq_length
				UpperCamelCase						=  is_training
				UpperCamelCase						=  use_input_mask
				UpperCamelCase						=  use_token_type_ids
				UpperCamelCase						=  use_labels
				UpperCamelCase						=  vocab_size
				UpperCamelCase						=  hidden_size
				UpperCamelCase						=  num_hidden_layers
				UpperCamelCase						=  num_attention_heads
				UpperCamelCase						=  intermediate_size
				UpperCamelCase						=  hidden_act
				UpperCamelCase						=  hidden_dropout_prob
				UpperCamelCase						=  attention_probs_dropout_prob
				UpperCamelCase						=  max_position_embeddings
				UpperCamelCase						=  type_vocab_size
				UpperCamelCase						=  type_sequence_label_size
				UpperCamelCase						=  initializer_range
				UpperCamelCase						=  num_labels
				UpperCamelCase						=  num_choices
				UpperCamelCase						=  scope



		def __UpperCamelCase				(				self )						->      Optional[int]:
				"""simple docstring"""
				UpperCamelCase						=  ids_tensor([self.batch_size, self.seq_length]       ,   self.vocab_size )

				UpperCamelCase						=  None
				if self.use_input_mask:
						UpperCamelCase						=  random_attention_mask([self.batch_size, self.seq_length] )

				UpperCamelCase						=  None
				if self.use_token_type_ids:
						UpperCamelCase						=  ids_tensor([self.batch_size, self.seq_length]       ,   self.type_vocab_size )

				UpperCamelCase						=  None
				UpperCamelCase						=  None
				UpperCamelCase						=  None
				if self.use_labels:
						UpperCamelCase						=  ids_tensor([self.batch_size]       ,   self.type_sequence_label_size )
						UpperCamelCase						=  ids_tensor([self.batch_size, self.seq_length]       ,   self.num_labels )
						UpperCamelCase						=  ids_tensor([self.batch_size]       ,   self.num_choices )

				UpperCamelCase						=  self.get_config()

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



		def __UpperCamelCase				(				self )						->      Dict:
				"""simple docstring"""
				return OpenLlamaConfig(
				    vocab_size=self.vocab_size       ,   hidden_size=self.hidden_size       ,   num_hidden_layers=self.num_hidden_layers       ,   num_attention_heads=self.num_attention_heads       ,   intermediate_size=self.intermediate_size       ,   hidden_act=self.hidden_act       ,   hidden_dropout_prob=self.hidden_dropout_prob       ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,   max_position_embeddings=self.max_position_embeddings       ,   type_vocab_size=self.type_vocab_size       ,   is_decoder=lowercase__       ,   initializer_range=self.initializer_range       ,   use_stable_embedding=lowercase__       ,   )



		def __UpperCamelCase				(				self       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_ )						->      List[str]:
				"""simple docstring"""
				UpperCamelCase						=  OpenLlamaModel(config=lowercase__ )
				model.to(lowercase__ )
				model.eval()
				UpperCamelCase						=  model(lowercase__       ,   attention_mask=lowercase__ )
				UpperCamelCase						=  model(lowercase__ )
				self.parent.assertEqual(result.last_hidden_state.shape       ,   (self.batch_size, self.seq_length, self.hidden_size) )



		def __UpperCamelCase				(				self       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   )						->      Tuple:
				"""simple docstring"""
				UpperCamelCase						=  True
				UpperCamelCase						=  OpenLlamaModel(lowercase__ )
				model.to(lowercase__ )
				model.eval()
				UpperCamelCase						=  model(
				    lowercase__       ,   attention_mask=lowercase__       ,   encoder_hidden_states=lowercase__       ,   encoder_attention_mask=lowercase__       ,   )
				UpperCamelCase						=  model(
				    lowercase__       ,   attention_mask=lowercase__       ,   encoder_hidden_states=lowercase__       ,   )
				UpperCamelCase						=  model(lowercase__       ,   attention_mask=lowercase__ )
				self.parent.assertEqual(result.last_hidden_state.shape       ,   (self.batch_size, self.seq_length, self.hidden_size) )



		def __UpperCamelCase				(				self       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   )						->      Dict:
				"""simple docstring"""
				UpperCamelCase						=  OpenLlamaForCausalLM(config=lowercase__ )
				model.to(lowercase__ )
				model.eval()
				UpperCamelCase						=  model(lowercase__       ,   attention_mask=lowercase__       ,   labels=lowercase__ )
				self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.seq_length, self.vocab_size) )



		def __UpperCamelCase				(				self       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   A_       ,   )						->      Optional[Any]:
				"""simple docstring"""
				UpperCamelCase						=  True
				UpperCamelCase						=  True
				UpperCamelCase						=  OpenLlamaForCausalLM(config=lowercase__ )
				model.to(lowercase__ )
				model.eval()

				# first forward pass
				UpperCamelCase						=  model(
				    lowercase__       ,   attention_mask=lowercase__       ,   encoder_hidden_states=lowercase__       ,   encoder_attention_mask=lowercase__       ,   use_cache=lowercase__       ,   )
				UpperCamelCase						=  outputs.past_key_values

				# create hypothetical multiple next token and extent to next_input_ids
				UpperCamelCase						=  ids_tensor((self.batch_size, 3)       ,   config.vocab_size )
				UpperCamelCase						=  ids_tensor((self.batch_size, 3)       ,   vocab_size=2 )

				# append to next input_ids and
				UpperCamelCase						=  torch.cat([input_ids, next_tokens]       ,   dim=-1 )
				UpperCamelCase						=  torch.cat([input_mask, next_mask]       ,   dim=-1 )

				UpperCamelCase						=  model(
				    lowercase__       ,   attention_mask=lowercase__       ,   encoder_hidden_states=lowercase__       ,   encoder_attention_mask=lowercase__       ,   output_hidden_states=lowercase__       ,   )["""hidden_states"""][0]
				UpperCamelCase						=  model(
				    lowercase__       ,   attention_mask=lowercase__       ,   encoder_hidden_states=lowercase__       ,   encoder_attention_mask=lowercase__       ,   past_key_values=lowercase__       ,   output_hidden_states=lowercase__       ,   )["""hidden_states"""][0]

				# select random slice
				UpperCamelCase						=  ids_tensor((1,)       ,   output_from_past.shape[-1] ).item()
				UpperCamelCase						=  output_from_no_past[:, -3:, random_slice_idx].detach()
				UpperCamelCase						=  output_from_past[:, :, random_slice_idx].detach()

				self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1] )

				# test that outputs are equal for slice
				self.parent.assertTrue(torch.allclose(lowercase__       ,   lowercase__       ,   atol=1e-3 ) )



		def __UpperCamelCase				(				self )						->      Union[str, Any]:
				"""simple docstring"""
				UpperCamelCase						=  self.prepare_config_and_inputs()
				(
				    UpperCamelCase			
				)						=  config_and_inputs
				UpperCamelCase						=  {"""input_ids""": input_ids, """attention_mask""": input_mask}
				return config, inputs_dict






@require_torch
class        lowercase      (	_UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     unittest.TestCase     ):
		__lowercase : Any			  =       (
		    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
		)
		__lowercase : Union[str, Any]			  =       (OpenLlamaForCausalLM,) if is_torch_available() else ()
		__lowercase : List[str]			  =       (
		    {
		        """feature-extraction""": OpenLlamaModel,
		        """text-classification""": OpenLlamaForSequenceClassification,
		        """text-generation""": OpenLlamaForCausalLM,
		        """zero-shot""": OpenLlamaForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		__lowercase : Optional[int]			  =       False
		__lowercase : Tuple			  =       False



		def __UpperCamelCase				(				self )						->      Optional[int]:
				"""simple docstring"""
				UpperCamelCase						=  OpenLlamaModelTester(self )
				UpperCamelCase						=  ConfigTester(self       ,   config_class=lowercase__       ,   hidden_size=37 )



		def __UpperCamelCase				(				self )						->      List[Any]:
				"""simple docstring"""
				self.config_tester.run_common_tests()



		def __UpperCamelCase				(				self )						->      Dict:
				"""simple docstring"""
				UpperCamelCase						=  self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*lowercase__ )



		def __UpperCamelCase				(				self )						->      Optional[int]:
				"""simple docstring"""
				UpperCamelCase						=  self.model_tester.prepare_config_and_inputs()
				for type in ["absolute", "relative_key", "relative_key_query"]:
						UpperCamelCase						=  type
						self.model_tester.create_and_check_model(*lowercase__ )



		def __UpperCamelCase				(				self )						->      List[Any]:
				"""simple docstring"""
				UpperCamelCase						=  self.model_tester.prepare_config_and_inputs_for_common()
				UpperCamelCase						=  3
				UpperCamelCase						=  input_dict["""input_ids"""]
				UpperCamelCase						=  input_ids.ne(1 ).to(lowercase__ )
				UpperCamelCase						=  ids_tensor([self.model_tester.batch_size]       ,   self.model_tester.type_sequence_label_size )
				UpperCamelCase						=  OpenLlamaForSequenceClassification(lowercase__ )
				model.to(lowercase__ )
				model.eval()
				UpperCamelCase						=  model(lowercase__       ,   attention_mask=lowercase__       ,   labels=lowercase__ )
				self.assertEqual(result.logits.shape       ,   (self.model_tester.batch_size, self.model_tester.num_labels) )



		def __UpperCamelCase				(				self )						->      Optional[Any]:
				"""simple docstring"""
				UpperCamelCase						=  self.model_tester.prepare_config_and_inputs_for_common()
				UpperCamelCase						=  3
				UpperCamelCase						=  """single_label_classification"""
				UpperCamelCase						=  input_dict["""input_ids"""]
				UpperCamelCase						=  input_ids.ne(1 ).to(lowercase__ )
				UpperCamelCase						=  ids_tensor([self.model_tester.batch_size]       ,   self.model_tester.type_sequence_label_size )
				UpperCamelCase						=  OpenLlamaForSequenceClassification(lowercase__ )
				model.to(lowercase__ )
				model.eval()
				UpperCamelCase						=  model(lowercase__       ,   attention_mask=lowercase__       ,   labels=lowercase__ )
				self.assertEqual(result.logits.shape       ,   (self.model_tester.batch_size, self.model_tester.num_labels) )



		def __UpperCamelCase				(				self )						->      Union[str, Any]:
				"""simple docstring"""
				UpperCamelCase						=  self.model_tester.prepare_config_and_inputs_for_common()
				UpperCamelCase						=  3
				UpperCamelCase						=  """multi_label_classification"""
				UpperCamelCase						=  input_dict["""input_ids"""]
				UpperCamelCase						=  input_ids.ne(1 ).to(lowercase__ )
				UpperCamelCase						=  ids_tensor(
				    [self.model_tester.batch_size, config.num_labels]       ,   self.model_tester.type_sequence_label_size ).to(torch.float )
				UpperCamelCase						=  OpenLlamaForSequenceClassification(lowercase__ )
				model.to(lowercase__ )
				model.eval()
				UpperCamelCase						=  model(lowercase__       ,   attention_mask=lowercase__       ,   labels=lowercase__ )
				self.assertEqual(result.logits.shape       ,   (self.model_tester.batch_size, self.model_tester.num_labels) )



		@unittest.skip('Open-Llama buffers include complex numbers, which breaks this test' )
		def __UpperCamelCase				(				self )						->      Any:
				"""simple docstring"""
				pass



		@parameterized.expand([('linear',), ('dynamic',)] )
		def __UpperCamelCase				(				self       ,   A_ )						->      Dict:
				"""simple docstring"""
				UpperCamelCase						=  self.model_tester.prepare_config_and_inputs_for_common()
				UpperCamelCase						=  ids_tensor([1, 10]       ,   config.vocab_size )
				UpperCamelCase						=  ids_tensor([1, int(config.max_position_embeddings * 1.5 )]       ,   config.vocab_size )

				set_seed(42 )  # Fixed seed at init time so the two models get the same random weights
				UpperCamelCase						=  OpenLlamaModel(lowercase__ )
				original_model.to(lowercase__ )
				original_model.eval()
				UpperCamelCase						=  original_model(lowercase__ ).last_hidden_state
				UpperCamelCase						=  original_model(lowercase__ ).last_hidden_state

				set_seed(42 )  # Fixed seed at init time so the two models get the same random weights
				UpperCamelCase						=  {"""type""": scaling_type, """factor""": 10.0}
				UpperCamelCase						=  OpenLlamaModel(lowercase__ )
				scaled_model.to(lowercase__ )
				scaled_model.eval()
				UpperCamelCase						=  scaled_model(lowercase__ ).last_hidden_state
				UpperCamelCase						=  scaled_model(lowercase__ ).last_hidden_state

				# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
				# maximum sequence length, so the outputs for the short input should match.
				if scaling_type == "dynamic":
						self.assertTrue(torch.allclose(lowercase__       ,   lowercase__       ,   atol=1e-5 ) )
				else:
						self.assertFalse(torch.allclose(lowercase__       ,   lowercase__       ,   atol=1e-5 ) )

				# The output should be different for long inputs
				self.assertFalse(torch.allclose(lowercase__       ,   lowercase__       ,   atol=1e-5 ) )





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_camembert import CamembertTokenizer
else:
					_UpperCAmelCase      :				List[str]          = None


_UpperCAmelCase      :				Any          = logging.get_logger(__name__)

_UpperCAmelCase      :				Tuple          = {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

_UpperCAmelCase      :				List[str]          = {
    "vocab_file": {
        "camembert-base": "https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_file": {
        "camembert-base": "https://huggingface.co/camembert-base/resolve/main/tokenizer.json",
    },
}

_UpperCAmelCase      :				Optional[int]          = {
    "camembert-base": 512,
}

_UpperCAmelCase      :				Union[str, Any]          = "▁"


class        lowercase      (	_SCREAMING_SNAKE_CASE     ):
		__lowercase : str			  =       VOCAB_FILES_NAMES
		__lowercase : Optional[Any]			  =       PRETRAINED_VOCAB_FILES_MAP
		__lowercase : List[str]			  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__lowercase : List[str]			  =       ["input_ids", "attention_mask"]
		__lowercase : Tuple			  =       CamembertTokenizer



		def __init__(				self       ,   A_=None       ,   A_=None       ,   A_="<s>"       ,   A_="</s>"       ,   A_="</s>"       ,   A_="<s>"       ,   A_="<unk>"       ,   A_="<pad>"       ,   A_="<mask>"       ,   A_=["<s>NOTUSED", "</s>NOTUSED"]       ,   **A_       ,   )						->      List[Any]:
				"""simple docstring"""
				# Mask token behave like a normal word, i.e. include the space before it
				UpperCamelCase						=  AddedToken(A_       ,   lstrip=A_       ,   rstrip=A_ ) if isinstance(A_       ,   A_ ) else mask_token

				super().__init__(
				    A_       ,   tokenizer_file=A_       ,   bos_token=A_       ,   eos_token=A_       ,   sep_token=A_       ,   cls_token=A_       ,   unk_token=A_       ,   pad_token=A_       ,   mask_token=A_       ,   additional_special_tokens=A_       ,   **A_       ,   )

				UpperCamelCase						=  vocab_file
				UpperCamelCase						=  False if not self.vocab_file else True



		def __UpperCamelCase				(				self       ,   A_       ,   A_ = None )						->      List[int]:
				"""simple docstring"""

				if token_ids_a is None:
						return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				UpperCamelCase						=  [self.cls_token_id]
				UpperCamelCase						=  [self.sep_token_id]
				return cls + token_ids_a + sep + sep + token_ids_a + sep



		def __UpperCamelCase				(				self       ,   A_       ,   A_ = None )						->      List[int]:
				"""simple docstring"""
				UpperCamelCase						=  [self.sep_token_id]
				UpperCamelCase						=  [self.cls_token_id]

				if token_ids_a is None:
						return len(cls + token_ids_a + sep ) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]



		def __UpperCamelCase				(				self       ,   A_       ,   A_ = None )						->      Tuple[str]:
				"""simple docstring"""
				if not self.can_save_slow_tokenizer:
						raise ValueError(
						    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
						    'tokenizer.' )

				if not os.path.isdir(A_ ):
						logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
						return
				UpperCamelCase						=  os.path.join(
				    A_       ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )

				if os.path.abspath(self.vocab_file ) != os.path.abspath(A_ ):
						copyfile(self.vocab_file       ,   A_ )

				return (out_vocab_file,)
