

import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE_					:	Optional[int]							  =						get_tests_dir('fixtures/test_sentencepiece.model')
SCREAMING_SNAKE_CASE_					:	Tuple							  =						get_tests_dir('fixtures/test_sentencepiece_bpe.model')

SCREAMING_SNAKE_CASE_					:	int							  =						'pt' if is_torch_available() else 'tf'



@require_sentencepiece
@require_tokenizers
class 					a						(       lowerCAmelCase__,	unittest.TestCase				):



    """simple docstring"""



    UpperCAmelCase						 =					CamembertTokenizer
    UpperCAmelCase						 =					CamembertTokenizerFast
    UpperCAmelCase						 =					True
    UpperCAmelCase						 =					True

    def 				UpperCamelCase	(     self: Tuple       ):






         """simple docstring"""


         super().setUp()

         # We have a SentencePiece fixture for testing
         A__										=  CamembertTokenizer(_SCREAMING_SNAKE_CASE       )
         tokenizer.save_pretrained(self.tmpdirname       )

    def 				UpperCamelCase	(     self: Union[str, Any]       ):






         """simple docstring"""


         A__										=  """<pad>"""
         A__										=  1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE       )    ,						_SCREAMING_SNAKE_CASE       )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE       )    ,						_SCREAMING_SNAKE_CASE       )

    def 				UpperCamelCase	(     self: Dict       ):






         """simple docstring"""


         A__										=  list(self.get_tokenizer().get_vocab().keys()       )

         self.assertEqual(vocab_keys[0]    ,						"""<s>NOTUSED"""       )
         self.assertEqual(vocab_keys[1]    ,						"""<pad>"""       )
         self.assertEqual(vocab_keys[-1]    ,						"""<mask>"""       )
         self.assertEqual(len(_SCREAMING_SNAKE_CASE       )    ,						10_04       )

    def 				UpperCamelCase	(     self: Union[str, Any]       ):






         """simple docstring"""


         self.assertEqual(self.get_tokenizer().vocab_size    ,						10_05       )

    def 				UpperCamelCase	(     self: Union[str, Any]       ):






         """simple docstring"""


         A__										=  CamembertTokenizer(_SCREAMING_SNAKE_CASE       )
         tokenizer.save_pretrained(self.tmpdirname       )
         A__										=  CamembertTokenizerFast.from_pretrained(self.tmpdirname       )

         A__										=  """I was born in 92000, and this is falsé."""

         A__										=  tokenizer.encode(_SCREAMING_SNAKE_CASE       )
         A__										=  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE       )
         self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE       )

         A__										=  tokenizer.encode(_SCREAMING_SNAKE_CASE    ,						add_special_tokens=_SCREAMING_SNAKE_CASE       )
         A__										=  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE    ,						add_special_tokens=_SCREAMING_SNAKE_CASE       )
         self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE       )

         # <unk> tokens are not the same for `rust` than for `slow`.
         # Because spm gives back raw token instead of `unk` in EncodeAsPieces
         # tokens = tokenizer.tokenize(sequence)
         A__										=  tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE       )
         A__										=  rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE       )
         self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE       )

    def 				UpperCamelCase	(     self: Dict       ):






         """simple docstring"""


         if not self.test_rust_tokenizer:
              return

         A__										=  self.get_tokenizer()
         A__										=  self.get_rust_tokenizer()

         A__										=  """I was born in 92000, and this is falsé."""

         A__										=  tokenizer.tokenize(_SCREAMING_SNAKE_CASE       )
         A__										=  rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE       )
         self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE       )

         A__										=  tokenizer.encode(_SCREAMING_SNAKE_CASE    ,						add_special_tokens=_SCREAMING_SNAKE_CASE       )
         A__										=  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE    ,						add_special_tokens=_SCREAMING_SNAKE_CASE       )
         self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE       )

         A__										=  self.get_rust_tokenizer()
         A__										=  tokenizer.encode(_SCREAMING_SNAKE_CASE       )
         A__										=  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE       )
         self.assertListEqual(_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE       )





    @slow
    def 				UpperCamelCase	(     self: List[Any]       ):






         """simple docstring"""


         A__										=  {"""input_ids""": [[5, 54, 71_96, 2_97, 30, 23, 7_76, 18, 11, 32_15, 37_05, 82_52, 22, 31_64, 11_81, 21_16, 29, 16, 8_13, 25, 7_91, 33_14, 20, 34_46, 38, 2_75_75, 1_20, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 4_68, 17, 11, 90_88, 20, 15_17, 8, 2_28_04, 1_88_18, 10, 38, 6_29, 6_07, 6_07, 1_42, 19, 71_96, 8_67, 56, 1_03_26, 24, 22_67, 20, 4_16, 50_72, 1_56_12, 2_33, 7_34, 7, 23_99, 27, 16, 30_15, 16_49, 7, 24, 20, 43_38, 23_99, 27, 13, 34_00, 14, 13, 61_89, 8, 9_30, 9, 6]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         # camembert is a french model. So we also use french texts.
         A__										=  [
             """Le transformeur est un modèle d'apprentissage profond introduit en 2017, """
             """utilisé principalement dans le domaine du traitement automatique des langues (TAL).""",
             """À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus """
             """pour gérer des données séquentielles, telles que le langage naturel, pour des tâches """
             """telles que la traduction et la synthèse de texte.""",
         ]

         self.tokenizer_integration_test_util(
             expected_encoding=_SCREAMING_SNAKE_CASE    ,						model_name="""camembert-base"""    ,						revision="""3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"""    ,						sequences=_SCREAMING_SNAKE_CASE    ,						)









"""simple docstring"""



import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
 import torch



@is_pipeline_test
class 					a						(       unittest.TestCase				):



    """simple docstring"""



    UpperCAmelCase						 =					MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
    UpperCAmelCase						 =					TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

    def 				UpperCamelCase	(     self: Dict    ,						UpperCamelCase: str    ,						UpperCamelCase: Optional[Any]    ,						UpperCamelCase: Any       ):






         """simple docstring"""


         A__										=  TextaTextGenerationPipeline(model=UpperCamelCase    ,						tokenizer=UpperCamelCase       )
         return generator, ["Something to write", "Something else"]

    def 				UpperCamelCase	(     self: List[Any]    ,						UpperCamelCase: Any    ,						UpperCamelCase: Dict       ):






         """simple docstring"""


         A__										=  generator("""Something there"""       )
         self.assertEqual(UpperCamelCase    ,						[{"""generated_text""": ANY(UpperCamelCase       )}]       )
         # These are encoder decoder, they don't just append to incoming string
         self.assertFalse(outputs[0]["""generated_text"""].startswith("""Something there"""       )       )

         A__										=  generator(["""This is great !""", """Something else"""]    ,						num_return_sequences=2    ,						do_sample=UpperCamelCase       )
         self.assertEqual(
             UpperCamelCase    ,						[
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
             ]    ,						)

         A__										=  generator(
             ["""This is great !""", """Something else"""]    ,						num_return_sequences=2    ,						batch_size=2    ,						do_sample=UpperCamelCase       )
         self.assertEqual(
             UpperCamelCase    ,						[
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
             ]    ,						)

         with self.assertRaises(UpperCamelCase       ):
              generator(4       )

    @require_torch
    def 				UpperCamelCase	(     self: Optional[Any]       ):






         """simple docstring"""


         A__										=  pipeline("""text2text-generation"""    ,						model="""patrickvonplaten/t5-tiny-random"""    ,						framework="""pt"""       )
         # do_sample=False necessary for reproducibility
         A__										=  generator("""Something there"""    ,						do_sample=UpperCamelCase       )
         self.assertEqual(UpperCamelCase    ,						[{"""generated_text""": """"""}]       )

         A__										=  3
         A__										=  generator(
             """Something there"""    ,						num_return_sequences=UpperCamelCase    ,						num_beams=UpperCamelCase    ,						)
         A__										=  [
             {"""generated_text""": """Beide Beide Beide Beide Beide Beide Beide Beide Beide"""},
             {"""generated_text""": """Beide Beide Beide Beide Beide Beide Beide Beide"""},
             {"""generated_text""": """"""},
         ]
         self.assertEqual(UpperCamelCase    ,						UpperCamelCase       )

         A__										=  generator("""This is a test"""    ,						do_sample=UpperCamelCase    ,						num_return_sequences=2    ,						return_tensors=UpperCamelCase       )
         self.assertEqual(
             UpperCamelCase    ,						[
                 {"""generated_token_ids""": ANY(torch.Tensor       )},
                 {"""generated_token_ids""": ANY(torch.Tensor       )},
             ]    ,						)
         A__										=  generator.model.config.eos_token_id
         A__										=  """<pad>"""
         A__										=  generator(
             ["""This is a test""", """This is a second test"""]    ,						do_sample=UpperCamelCase    ,						num_return_sequences=2    ,						batch_size=2    ,						return_tensors=UpperCamelCase    ,						)
         self.assertEqual(
             UpperCamelCase    ,						[
                 [
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                 ],
                 [
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                 ],
             ]    ,						)





    @require_tf
    def 				UpperCamelCase	(     self: Any       ):






         """simple docstring"""


         A__										=  pipeline("""text2text-generation"""    ,						model="""patrickvonplaten/t5-tiny-random"""    ,						framework="""tf"""       )
         # do_sample=False necessary for reproducibility
         A__										=  generator("""Something there"""    ,						do_sample=UpperCamelCase       )
         self.assertEqual(UpperCamelCase    ,						[{"""generated_text""": """"""}]       )






