

import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


SCREAMING_SNAKE_CASE_					:	Any							  =						5_0_0_0_0_0

SCREAMING_SNAKE_CASE_					:	List[Any]							  =						os.path.split(__file__)
SCREAMING_SNAKE_CASE_					:	Optional[int]							  =						os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))





@get_duration
def  _snake_case     (    UpperCAmelCase_  :					datasets.Dataset					,   **UpperCAmelCase_  :					Optional[int]			):
     A__										=  dataset.map(**UpperCAmelCase_			)





@get_duration
def  _snake_case     (    UpperCAmelCase_  :					datasets.Dataset					,   **UpperCAmelCase_  :					Tuple			):
     A__										=  dataset.filter(**UpperCAmelCase_			)





def  _snake_case     (    ):
     A__										=  {"""num examples""": SPEED_TEST_N_EXAMPLES}
     with tempfile.TemporaryDirectory() as tmp_dir:
          A__										=  datasets.Features({"""text""": datasets.Value("""string"""			), """numbers""": datasets.Value("""float32"""			)}			)
          A__										=  generate_example_dataset(
              os.path.join(UpperCAmelCase_					,   """dataset.arrow"""			)					,   UpperCAmelCase_					,   num_examples=UpperCAmelCase_			)

          A__										=  transformers.AutoTokenizer.from_pretrained("""bert-base-cased"""					,   use_fast=UpperCAmelCase_			)

          def tokenize(UpperCAmelCase_  :					Dict			):
               return tokenizer(examples["""text"""]			)

          A__										=  map(UpperCAmelCase_			)

          A__										=  map(UpperCAmelCase_					,   batched=UpperCAmelCase_			)

          A__										=  map(UpperCAmelCase_					,   function=lambda UpperCAmelCase_			: None					,   batched=UpperCAmelCase_			)

          with dataset.formatted_as(type="""numpy"""			):
               A__										=  map(UpperCAmelCase_					,   function=lambda UpperCAmelCase_			: None					,   batched=UpperCAmelCase_			)

          with dataset.formatted_as(type="""pandas"""			):
               A__										=  map(UpperCAmelCase_					,   function=lambda UpperCAmelCase_			: None					,   batched=UpperCAmelCase_			)

          with dataset.formatted_as(type="""torch"""					,   columns="""numbers"""			):
               A__										=  map(UpperCAmelCase_					,   function=lambda UpperCAmelCase_			: None					,   batched=UpperCAmelCase_			)

          with dataset.formatted_as(type="""tensorflow"""					,   columns="""numbers"""			):
               A__										=  map(UpperCAmelCase_					,   function=lambda UpperCAmelCase_			: None					,   batched=UpperCAmelCase_			)

          A__										=  map(UpperCAmelCase_					,   function=UpperCAmelCase_					,   batched=UpperCAmelCase_			)

          A__										=  filter(UpperCAmelCase_			)

          # Activate later when tokenizer support batched inputs
          # with dataset.formatted_as(type='numpy'):
          #     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

     with open(UpperCAmelCase_					,   """wb"""			) as f:
          f.write(json.dumps(UpperCAmelCase_			).encode("""utf-8"""			)			)


if __name__ == "__main__":  # useful to run the profiler
 benchmark_map_filter()









"""simple docstring"""



import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
 import torch



@is_pipeline_test
class 					a						(       unittest.TestCase				):



    """simple docstring"""



    UpperCAmelCase						 =					MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
    UpperCAmelCase						 =					TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

    def 				UpperCamelCase	(     self: Dict    ,						UpperCamelCase: str    ,						UpperCamelCase: Optional[Any]    ,						UpperCamelCase: Any       ):






         """simple docstring"""


         A__										=  TextaTextGenerationPipeline(model=UpperCamelCase    ,						tokenizer=UpperCamelCase       )
         return generator, ["Something to write", "Something else"]

    def 				UpperCamelCase	(     self: List[Any]    ,						UpperCamelCase: Any    ,						UpperCamelCase: Dict       ):






         """simple docstring"""


         A__										=  generator("""Something there"""       )
         self.assertEqual(UpperCamelCase    ,						[{"""generated_text""": ANY(UpperCamelCase       )}]       )
         # These are encoder decoder, they don't just append to incoming string
         self.assertFalse(outputs[0]["""generated_text"""].startswith("""Something there"""       )       )

         A__										=  generator(["""This is great !""", """Something else"""]    ,						num_return_sequences=2    ,						do_sample=UpperCamelCase       )
         self.assertEqual(
             UpperCamelCase    ,						[
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
             ]    ,						)

         A__										=  generator(
             ["""This is great !""", """Something else"""]    ,						num_return_sequences=2    ,						batch_size=2    ,						do_sample=UpperCamelCase       )
         self.assertEqual(
             UpperCamelCase    ,						[
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
                 [{"""generated_text""": ANY(UpperCamelCase       )}, {"""generated_text""": ANY(UpperCamelCase       )}],
             ]    ,						)

         with self.assertRaises(UpperCamelCase       ):
              generator(4       )

    @require_torch
    def 				UpperCamelCase	(     self: Optional[Any]       ):






         """simple docstring"""


         A__										=  pipeline("""text2text-generation"""    ,						model="""patrickvonplaten/t5-tiny-random"""    ,						framework="""pt"""       )
         # do_sample=False necessary for reproducibility
         A__										=  generator("""Something there"""    ,						do_sample=UpperCamelCase       )
         self.assertEqual(UpperCamelCase    ,						[{"""generated_text""": """"""}]       )

         A__										=  3
         A__										=  generator(
             """Something there"""    ,						num_return_sequences=UpperCamelCase    ,						num_beams=UpperCamelCase    ,						)
         A__										=  [
             {"""generated_text""": """Beide Beide Beide Beide Beide Beide Beide Beide Beide"""},
             {"""generated_text""": """Beide Beide Beide Beide Beide Beide Beide Beide"""},
             {"""generated_text""": """"""},
         ]
         self.assertEqual(UpperCamelCase    ,						UpperCamelCase       )

         A__										=  generator("""This is a test"""    ,						do_sample=UpperCamelCase    ,						num_return_sequences=2    ,						return_tensors=UpperCamelCase       )
         self.assertEqual(
             UpperCamelCase    ,						[
                 {"""generated_token_ids""": ANY(torch.Tensor       )},
                 {"""generated_token_ids""": ANY(torch.Tensor       )},
             ]    ,						)
         A__										=  generator.model.config.eos_token_id
         A__										=  """<pad>"""
         A__										=  generator(
             ["""This is a test""", """This is a second test"""]    ,						do_sample=UpperCamelCase    ,						num_return_sequences=2    ,						batch_size=2    ,						return_tensors=UpperCamelCase    ,						)
         self.assertEqual(
             UpperCamelCase    ,						[
                 [
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                 ],
                 [
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                     {"""generated_token_ids""": ANY(torch.Tensor       )},
                 ],
             ]    ,						)





    @require_tf
    def 				UpperCamelCase	(     self: Any       ):






         """simple docstring"""


         A__										=  pipeline("""text2text-generation"""    ,						model="""patrickvonplaten/t5-tiny-random"""    ,						framework="""tf"""       )
         # do_sample=False necessary for reproducibility
         A__										=  generator("""Something there"""    ,						do_sample=UpperCamelCase       )
         self.assertEqual(UpperCamelCase    ,						[{"""generated_text""": """"""}]       )






