

import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
 SCREAMING_SNAKE_CASE_					:	Dict							  =						'pt'
elif is_tf_available():
 SCREAMING_SNAKE_CASE_					:	int							  =						'tf'
else:
 SCREAMING_SNAKE_CASE_					:	int							  =						'jax'



class 					a						(       __A,	unittest.TestCase				):



    """simple docstring"""



    UpperCAmelCase						 =					ByTaTokenizer
    UpperCAmelCase						 =					False

    def 				UpperCamelCase	(     self: Optional[int]       ):






         """simple docstring"""


         super().setUp()
         A__										=  ByTaTokenizer()
         tokenizer.save_pretrained(self.tmpdirname       )

    @cached_property
    def 				UpperCamelCase	(     self: List[str]       ):






         """simple docstring"""


         return ByTaTokenizer.from_pretrained("""google/byt5-small"""       )

    def 				UpperCamelCase	(     self: Optional[int]    ,						**UpperCamelCase: Optional[Any]       ):






         """simple docstring"""


         return self.tokenizer_class.from_pretrained(self.tmpdirname    ,						**UpperCamelCase__       )

    def 				UpperCamelCase	(     self: Dict    ,						UpperCamelCase: List[str]    ,						UpperCamelCase: Union[str, Any]=False    ,						UpperCamelCase: List[Any]=20    ,						UpperCamelCase: Any=5       ):






         """simple docstring"""


         A__										=  []
         for i in range(len(UpperCamelCase__       )       ):
              try:
                   A__										=  tokenizer.decode([i]    ,						clean_up_tokenization_spaces=UpperCamelCase__       )
              except UnicodeDecodeError:
                   pass
              toks.append((i, tok)       )

         A__										=  list(filter(lambda UpperCamelCase       : re.match(r"""^[ a-zA-Z]+$"""    ,						t[1]       )    ,						UpperCamelCase__       )       )
         A__										=  list(filter(lambda UpperCamelCase       : [t[0]] == tokenizer.encode(t[1]    ,						add_special_tokens=UpperCamelCase__       )    ,						UpperCamelCase__       )       )
         if max_length is not None and len(UpperCamelCase__       ) > max_length:
              A__										=  toks[:max_length]
         if min_length is not None and len(UpperCamelCase__       ) < min_length and len(UpperCamelCase__       ) > 0:
              while len(UpperCamelCase__       ) < min_length:
                   A__										=  toks + toks
        # toks_str = [t[1] for t in toks]
         A__										=  [t[0] for t in toks]

         # Ensure consistency
         A__										=  tokenizer.decode(UpperCamelCase__    ,						clean_up_tokenization_spaces=UpperCamelCase__       )
         if " " not in output_txt and len(UpperCamelCase__       ) > 1:
              A__										=  (
                  tokenizer.decode([toks_ids[0]]    ,						clean_up_tokenization_spaces=UpperCamelCase__       )
                  + ' '
                  + tokenizer.decode(toks_ids[1:]    ,						clean_up_tokenization_spaces=UpperCamelCase__       )
              )
         if with_prefix_space:
              A__										=  ' ' + output_txt
         A__										=  tokenizer.encode(UpperCamelCase__    ,						add_special_tokens=UpperCamelCase__       )
         return output_txt, output_ids

    def 				UpperCamelCase	(     self: List[Any]       ):






         """simple docstring"""


         A__										=  self.ta_base_tokenizer
         A__										=  tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]       )
         A__										=  tokenizer(["""hi""", """I went to the gym""", """"""]       )
         self.assertListEqual(batch_with_eos_added["""input_ids"""]    ,						batch_without_eos_added["""input_ids"""]       )

    def 				UpperCamelCase	(     self: Tuple       ):






         """simple docstring"""


         A__										=  self.ta_base_tokenizer
         A__										=  'Unicode €.'
         A__										=  tokenizer(UpperCamelCase__       )
         A__										=  [88, 1_13, 1_08, 1_02, 1_14, 1_03, 1_04, 35, 2_29, 1_33, 1_75, 49, 1]
         self.assertEqual(encoded["""input_ids"""]    ,						UpperCamelCase__       )

         # decoding
         A__										=  tokenizer.decode(UpperCamelCase__       )
         self.assertEqual(UpperCamelCase__    ,						"""Unicode €.</s>"""       )

         A__										=  tokenizer("""e è é ê ë"""       )
         A__										=  [1_04, 35, 1_98, 1_71, 35, 1_98, 1_72, 35, 1_98, 1_73, 35, 1_98, 1_74, 1]
         self.assertEqual(encoded["""input_ids"""]    ,						UpperCamelCase__       )
         # decoding
         A__										=  tokenizer.decode(UpperCamelCase__       )
         self.assertEqual(UpperCamelCase__    ,						"""e è é ê ë</s>"""       )

         # encode/decode, but with `encode` instead of `__call__`
         self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""       )       )    ,						"""e è é ê ë</s>"""       )

    def 				UpperCamelCase	(     self: Any       ):






         """simple docstring"""


         A__										=  self.ta_base_tokenizer
         A__										=  ['A long paragraph for summarization.', 'Another paragraph for summarization.']
         # fmt: off
         A__										=  [68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 1, 0]
         # fmt: on
         A__										=  tokenizer(UpperCamelCase__    ,						padding=UpperCamelCase__    ,						return_tensors=UpperCamelCase__       )
         self.assertIsInstance(UpperCamelCase__    ,						UpperCamelCase__       )

         if FRAMEWORK != "jax":
              A__										=  list(batch.input_ids.numpy()[0]       )
         else:
              A__										=  list(batch.input_ids.tolist()[0]       )

         self.assertListEqual(UpperCamelCase__    ,						UpperCamelCase__       )

         self.assertEqual((2, 37)    ,						batch.input_ids.shape       )
         self.assertEqual((2, 37)    ,						batch.attention_mask.shape       )

    def 				UpperCamelCase	(     self: Any       ):






         """simple docstring"""


         A__										=  self.ta_base_tokenizer
         A__										=  ['A long paragraph for summarization.', 'Another paragraph for summarization.']
         A__										=  tokenizer(UpperCamelCase__    ,						padding=UpperCamelCase__    ,						return_tensors=UpperCamelCase__       )
         # check if input_ids are returned and no decoder_input_ids
         self.assertIn("""input_ids"""    ,						UpperCamelCase__       )
         self.assertIn("""attention_mask"""    ,						UpperCamelCase__       )
         self.assertNotIn("""decoder_input_ids"""    ,						UpperCamelCase__       )
         self.assertNotIn("""decoder_attention_mask"""    ,						UpperCamelCase__       )

    def 				UpperCamelCase	(     self: Optional[Any]       ):






         """simple docstring"""


         A__										=  self.ta_base_tokenizer
         A__										=  [
             'Summary of the text.',
             'Another summary.',
         ]
         A__										=  tokenizer(
             text_target=UpperCamelCase__    ,						max_length=32    ,						padding="""max_length"""    ,						truncation=UpperCamelCase__    ,						return_tensors=UpperCamelCase__       )
         self.assertEqual(32    ,						targets["""input_ids"""].shape[1]       )

    def 				UpperCamelCase	(     self: List[Any]       ):






         """simple docstring"""


         A__										=  self.ta_base_tokenizer
         A__										=  ['A long paragraph for summarization. </s>']
         A__										=  ['Summary of the text. </s>']
         # fmt: off
         A__										=  [68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 35, 1]
         A__										=  [86, 1_20, 1_12, 1_12, 1_00, 1_17, 1_24, 35, 1_14, 1_05, 35, 1_19, 1_07, 1_04, 35, 1_19, 1_04, 1_23, 1_19, 49, 35, 1]
         # fmt: on

         A__										=  tokenizer(UpperCamelCase__    ,						text_target=UpperCamelCase__       )

         self.assertEqual(UpperCamelCase__    ,						batch["""input_ids"""][0]       )
         self.assertEqual(UpperCamelCase__    ,						batch["""labels"""][0]       )

    def 				UpperCamelCase	(     self: str       ):






         """simple docstring"""


         A__										=  self.get_tokenizers()
         for tokenizer in tokenizers:
              with self.subTest(f"""{tokenizer.__class__.__name__}"""       ):
                   self.assertNotEqual(tokenizer.model_max_length    ,						42       )

        # Now let's start the test
         A__										=  self.get_tokenizers()
         for tokenizer in tokenizers:
              with self.subTest(f"""{tokenizer.__class__.__name__}"""       ):
                   # Isolate this from the other tests because we save additional tokens/etc
                   A__										=  tempfile.mkdtemp()

                   A__										=  ' He is very happy, UNwant\u00E9d,running'
                   A__										=  tokenizer.encode(UpperCamelCase__    ,						add_special_tokens=UpperCamelCase__       )
                   tokenizer.save_pretrained(UpperCamelCase__       )

                   A__										=  tokenizer.__class__.from_pretrained(UpperCamelCase__       )
                   A__										=  after_tokenizer.encode(UpperCamelCase__    ,						add_special_tokens=UpperCamelCase__       )
                   self.assertListEqual(UpperCamelCase__    ,						UpperCamelCase__       )

                   shutil.rmtree(UpperCamelCase__       )

         A__										=  self.get_tokenizers(model_max_length=42       )
         for tokenizer in tokenizers:
              with self.subTest(f"""{tokenizer.__class__.__name__}"""       ):
                   # Isolate this from the other tests because we save additional tokens/etc
                   A__										=  tempfile.mkdtemp()

                   A__										=  ' He is very happy, UNwant\u00E9d,running'
                   tokenizer.add_tokens(["""bim""", """bambam"""]       )
                   A__										=  tokenizer.additional_special_tokens
                   additional_special_tokens.append("""new_additional_special_token"""       )
                   tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}       )
                   A__										=  tokenizer.encode(UpperCamelCase__    ,						add_special_tokens=UpperCamelCase__       )
                   tokenizer.save_pretrained(UpperCamelCase__       )

                   A__										=  tokenizer.__class__.from_pretrained(UpperCamelCase__       )
                   A__										=  after_tokenizer.encode(UpperCamelCase__    ,						add_special_tokens=UpperCamelCase__       )
                   self.assertListEqual(UpperCamelCase__    ,						UpperCamelCase__       )
                   self.assertIn("""new_additional_special_token"""    ,						after_tokenizer.additional_special_tokens       )
                   self.assertEqual(after_tokenizer.model_max_length    ,						42       )

                   A__										=  tokenizer.__class__.from_pretrained(UpperCamelCase__    ,						model_max_length=43       )
                   self.assertEqual(tokenizer.model_max_length    ,						43       )

                   shutil.rmtree(UpperCamelCase__       )

    def 				UpperCamelCase	(     self: Dict       ):






         """simple docstring"""


         A__										=  []
         if self.test_slow_tokenizer:
              tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

         if self.test_rust_tokenizer:
              tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
              with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer_utils.save_pretrained(UpperCamelCase__       )

                   with open(os.path.join(UpperCamelCase__    ,						"""special_tokens_map.json"""       )    ,						encoding="""utf-8"""       ) as json_file:
                        A__										=  json.load(UpperCamelCase__       )

                   with open(os.path.join(UpperCamelCase__    ,						"""tokenizer_config.json"""       )    ,						encoding="""utf-8"""       ) as json_file:
                        A__										=  json.load(UpperCamelCase__       )

                   A__										=  [f"""<extra_id_{i}>""" for i in range(1_25       )]

                   A__										=  added_tokens_extra_ids + [
                       'an_additional_special_token'
                   ]
                   A__										=  added_tokens_extra_ids + [
                       'an_additional_special_token'
                   ]

                   with open(os.path.join(UpperCamelCase__    ,						"""special_tokens_map.json"""       )    ,						"""w"""    ,						encoding="""utf-8"""       ) as outfile:
                        json.dump(UpperCamelCase__    ,						UpperCamelCase__       )
                   with open(os.path.join(UpperCamelCase__    ,						"""tokenizer_config.json"""       )    ,						"""w"""    ,						encoding="""utf-8"""       ) as outfile:
                        json.dump(UpperCamelCase__    ,						UpperCamelCase__       )

                   # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                   # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                   # "special_tokens_map.json" files
                   A__										=  tokenizer_class.from_pretrained(
                       UpperCamelCase__    ,						)
                   self.assertIn(
                       """an_additional_special_token"""    ,						tokenizer_without_change_in_init.additional_special_tokens       )
                   # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                   self.assertEqual(
                       ["""an_additional_special_token"""]    ,						tokenizer_without_change_in_init.convert_ids_to_tokens(
                           tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]       )       )    ,						)

                   # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                   A__										=  added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""    ,						lstrip=UpperCamelCase__       )]
                   A__										=  tokenizer_class.from_pretrained(
                       UpperCamelCase__    ,						additional_special_tokens=UpperCamelCase__    ,						)

                   self.assertIn("""a_new_additional_special_token"""    ,						tokenizer.additional_special_tokens       )
                   self.assertEqual(
                       ["""a_new_additional_special_token"""]    ,						tokenizer.convert_ids_to_tokens(
                           tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]       )       )    ,						)

    def 				UpperCamelCase	(     self: List[str]       ):






         """simple docstring"""


         A__										=  []
         if self.test_slow_tokenizer:
              tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

         if self.test_rust_tokenizer:
              tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
              with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer_utils.save_pretrained(UpperCamelCase__       )

                   A__										=  tokenizer_class.from_pretrained(UpperCamelCase__       )

                   self.assertTrue(tokenizer.decode([2_55]       ) == """"""       )

    def 				UpperCamelCase	(     self: Optional[Any]       ):






         """simple docstring"""


         pass

    def 				UpperCamelCase	(     self: Optional[int]       ):






         """simple docstring"""


         pass

    def 				UpperCamelCase	(     self: str       ):






         """simple docstring"""


         pass

    def 				UpperCamelCase	(     self: List[str]       ):






         """simple docstring"""


         pass

    def 				UpperCamelCase	(     self: int       ):






         """simple docstring"""


         A__										=  self.get_tokenizers(fast=UpperCamelCase__    ,						do_lower_case=UpperCamelCase__       )
         for tokenizer in tokenizers:
              with self.subTest(f"""{tokenizer.__class__.__name__}"""       ):
                   A__										=  ['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
                   A__										=  tokenizer.convert_tokens_to_string(UpperCamelCase__       )

                   self.assertIsInstance(UpperCamelCase__    ,						UpperCamelCase__       )





    def 				UpperCamelCase	(     self: int       ):






         """simple docstring"""


         A__										=  self.get_tokenizers()
         for tokenizer in tokenizers:
              with self.subTest(f"""{tokenizer.__class__.__name__}"""       ):
                   A__										=  [
                       'bos_token',
                       'eos_token',
                       'unk_token',
                       'sep_token',
                       'pad_token',
                       'cls_token',
                       'mask_token',
                   ]

                   A__										=  0
                   A__										=  tokenizer.convert_ids_to_tokens(
                       UpperCamelCase__    ,						skip_special_tokens=UpperCamelCase__       )

                   for attr in attributes_list:
                        setattr(UpperCamelCase__    ,						attr + """_id"""    ,						UpperCamelCase__       )
                        self.assertEqual(getattr(UpperCamelCase__    ,						UpperCamelCase__       )    ,						UpperCamelCase__       )
                        self.assertEqual(getattr(UpperCamelCase__    ,						attr + """_id"""       )    ,						UpperCamelCase__       )

                        setattr(UpperCamelCase__    ,						attr + """_id"""    ,						UpperCamelCase__       )
                        self.assertEqual(getattr(UpperCamelCase__    ,						UpperCamelCase__       )    ,						UpperCamelCase__       )
                        self.assertEqual(getattr(UpperCamelCase__    ,						attr + """_id"""       )    ,						UpperCamelCase__       )

                   setattr(UpperCamelCase__    ,						"""additional_special_tokens_ids"""    ,						[]       )
                   self.assertListEqual(getattr(UpperCamelCase__    ,						"""additional_special_tokens"""       )    ,						[]       )
                   self.assertListEqual(getattr(UpperCamelCase__    ,						"""additional_special_tokens_ids"""       )    ,						[]       )

                   setattr(UpperCamelCase__    ,						"""additional_special_tokens_ids"""    ,						[token_id_to_test_setters]       )
                   self.assertListEqual(getattr(UpperCamelCase__    ,						"""additional_special_tokens"""       )    ,						[token_to_test_setters]       )
                   self.assertListEqual(getattr(UpperCamelCase__    ,						"""additional_special_tokens_ids"""       )    ,						[token_id_to_test_setters]       )









"""simple docstring"""



import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
 import torch



@is_pipeline_test
class 					a						(       unittest.TestCase				):



    """simple docstring"""



    UpperCAmelCase						 =					MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
    UpperCAmelCase						 =					TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

    def 				UpperCamelCase	(     self: Dict    ,						UpperCamelCase: str    ,						UpperCamelCase: Optional[Any]    ,						UpperCamelCase: Any       ):






         """simple docstring"""


         A__										=  TextaTextGenerationPipeline(model=UpperCamelCase    ,						tokenizer=UpperCamelCase       )
         return generator, ["Something to write", "Something else"]

    def 				UpperCamelCase	(     self: List[Any]    ,						UpperCamelCase: Any    ,						UpperCamelCase: Dict       ):






         """simple docstring"""


         A__										=  generator("""Something there"""       )
         self.assertEqual(UpperCamelCase    ,						[{"""generated_text""": ANY(UpperCamelCase       )}]       )
         # These are encoder decoder, they don't just append to incoming string
         self.assertFalse(outputs[0]["""generated_text"""].startswith("""Something there"""       )       )

         A__										=  generator(["""This is great !""", """Something else"""]    ,						num_return_sequences=2    ,						do_sample=UpperCamelCase       )
         self.assertEqual(
             UpperCamelCase    ,						[
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
             ]    ,						)

         A__										=  generator(
             ["""This is great !""", """Something else"""]    ,						num_return_sequences=2    ,						batch_size=2    ,						do_sample=UpperCamelCase       )
         self.assertEqual(
             UpperCamelCase    ,						[
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
             ]    ,						)

         with self.assertRaises(UpperCamelCase       ):
              generator(4       )

    @require_torch
    def 				UpperCamelCase	(     self: Optional[Any]       ):






         """simple docstring"""


         A__										=  pipeline("""text2text-generation"""    ,						model="""patrickvonplaten/t5-tiny-random"""    ,						framework="""pt"""       )
         # do_sample=False necessary for reproducibility
         A__										=  generator("""Something there"""    ,						do_sample=UpperCamelCase       )
         self.assertEqual(UpperCamelCase    ,						[{"""generated_text""": """"""}]       )

         A__										=  3
         A__										=  generator(
             """Something there"""    ,						num_return_sequences=UpperCamelCase    ,						num_beams=UpperCamelCase    ,						)
         A__										=  [
             {"""generated_text""": """Beide Beide Beide Beide Beide Beide Beide Beide Beide"""},
             {"""generated_text""": """Beide Beide Beide Beide Beide Beide Beide Beide"""},
             {"""generated_text""": """"""},
         ]
         self.assertEqual(UpperCamelCase    ,						UpperCamelCase       )

         A__										=  generator("""This is a test"""    ,						do_sample=UpperCamelCase    ,						num_return_sequences=2    ,						return_tensors=UpperCamelCase       )
         self.assertEqual(
             UpperCamelCase    ,						[
                 {"""generated_token_ids""": ANY(torch.Tensor       )},
                 {"""generated_token_ids""": ANY(torch.Tensor       )},
             ]    ,						)
         A__										=  generator.model.config.eos_token_id
         A__										=  """<pad>"""
         A__										=  generator(
             ["""This is a test""", """This is a second test"""]    ,						do_sample=UpperCamelCase    ,						num_return_sequences=2    ,						batch_size=2    ,						return_tensors=UpperCamelCase    ,						)
         self.assertEqual(
             UpperCamelCase    ,						[
                 [
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                 ],
                 [
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                 ],
             ]    ,						)





    @require_tf
    def 				UpperCamelCase	(     self: Any       ):






         """simple docstring"""


         A__										=  pipeline("""text2text-generation"""    ,						model="""patrickvonplaten/t5-tiny-random"""    ,						framework="""tf"""       )
         # do_sample=False necessary for reproducibility
         A__										=  generator("""Something there"""    ,						do_sample=UpperCamelCase       )
         self.assertEqual(UpperCamelCase    ,						[{"""generated_text""": """"""}]       )






