
'''simple docstring'''
import numpy as np







def        A_     (						__SCREAMING_SNAKE_CASE						:  int			,     __SCREAMING_SNAKE_CASE						:  List[Any]			,     __SCREAMING_SNAKE_CASE						:  Dict			,     __SCREAMING_SNAKE_CASE						:  int			,     __SCREAMING_SNAKE_CASE						:  Any )				->       List[str]:





  """simple docstring"""

  __A		:     Dict												=		int(np.ceil((x_end - xa) / h ) )
  __A		:     List[Any]												=		np.zeros((n + 1,) )
  __A		:     List[str]												=		ya
  __A		:     Any												=		xa

  for k in range(__SCREAMING_SNAKE_CASE ):
    __A		:     Optional[Any]												=		f(__SCREAMING_SNAKE_CASE			,     y[k] )
    __A		:     List[str]												=		f(x + 0.5 * h			,     y[k] + 0.5 * h * ka )
    __A		:     Optional[Any]												=		f(x + 0.5 * h			,     y[k] + 0.5 * h * ka )
    __A		:     Optional[Any]												=		f(x + h			,     y[k] + h * ka )
    __A		:     Tuple												=		y[k] + (1 / 6) * h * (ka + 2 * ka + 2 * ka + ka)
    x += h

  return y


if __name__ == "__main__":
     import doctest

     doctest.testmod()





'''simple docstring'''
import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
     A__   :		List[str] 		='pt'
elif is_tf_available():
     A__   :		List[str] 		='tf'
else:
     A__   :		List[str] 		='jax'







class  __A				( _SCREAMING_SNAKE_CASE    ,						unittest.TestCase			):
 lowerCamelCase				       =PerceiverTokenizer
 lowerCamelCase				       =False
 def 		lowercase_(   self			:    int       ):



   """simple docstring"""





   super().setUp()
   __A		:     Optional[int]												=		PerceiverTokenizer()
   tokenizer.save_pretrained(self.tmpdirname       )
 @cached_property
 def 		lowercase_(   self			:    Any       ):



   """simple docstring"""





   return PerceiverTokenizer.from_pretrained("""deepmind/language-perceiver"""       )
 def 		lowercase_(   self			:    Optional[Any]				,   **lowerCamelCase			:    Union[str, Any]       ):



   """simple docstring"""





   return self.tokenizer_class.from_pretrained(self.tmpdirname				,   **lowerCamelCase       )
 def 		lowercase_(   self			:    Any				,   lowerCamelCase			:    List[Any]				,   lowerCamelCase			:    Any=False				,   lowerCamelCase			:    List[Any]=20				,   lowerCamelCase			:    List[str]=5       ):



   """simple docstring"""





   __A		:     Optional[Any]												=		[]
   for i in range(len(lowerCamelCase       )       ):
     try:
       __A		:     str												=		tokenizer.decode([i]				,   clean_up_tokenization_spaces=lowerCamelCase       )
     except UnicodeDecodeError:
       pass
     toks.append((i, tok)       )

   __A		:     List[str]												=		list(filter(lambda lowerCamelCase       : re.match(r"""^[ a-zA-Z]+$"""				,   t[1]       )				,   lowerCamelCase       )       )
   __A		:     List[str]												=		list(filter(lambda lowerCamelCase       : [t[0]] == tokenizer.encode(t[1]				,   add_special_tokens=lowerCamelCase       )				,   lowerCamelCase       )       )
   if max_length is not None and len(lowerCamelCase       ) > max_length:
     __A		:     Optional[Any]												=		toks[:max_length]
   if min_length is not None and len(lowerCamelCase       ) < min_length and len(lowerCamelCase       ) > 0:
     while len(lowerCamelCase       ) < min_length:
       __A		:     Dict												=		toks + toks
        # toks_str = [t[1] for t in toks]
   __A		:     Optional[int]												=		[t[0] for t in toks]

   # Ensure consistency
   __A		:     Optional[Any]												=		tokenizer.decode(lowerCamelCase				,   clean_up_tokenization_spaces=lowerCamelCase       )
   if " " not in output_txt and len(lowerCamelCase       ) > 1:
     __A		:     int												=		(
         tokenizer.decode([toks_ids[0]]				,   clean_up_tokenization_spaces=lowerCamelCase       )
         + """ """
         + tokenizer.decode(toks_ids[1:]				,   clean_up_tokenization_spaces=lowerCamelCase       )
     )
   if with_prefix_space:
     __A		:     Union[str, Any]												=		""" """ + output_txt
   __A		:     Any												=		tokenizer.encode(lowerCamelCase				,   add_special_tokens=lowerCamelCase       )
   return output_txt, output_ids
 def 		lowercase_(   self			:    Optional[Any]       ):



   """simple docstring"""





   __A		:     int												=		self.perceiver_tokenizer
   __A		:     List[str]												=		"""Unicode €."""
   __A		:     Dict												=		tokenizer(lowerCamelCase       )
   __A		:     Any												=		[4, 91, 1_16, 1_11, 1_05, 1_17, 1_06, 1_07, 38, 2_32, 1_36, 1_78, 52, 5]
   self.assertEqual(encoded["""input_ids"""]				,   lowerCamelCase       )

   # decoding
   __A		:     Dict												=		tokenizer.decode(lowerCamelCase       )
   self.assertEqual(lowerCamelCase				,   """[CLS]Unicode €.[SEP]"""       )

   __A		:     int												=		tokenizer("""e è é ê ë"""       )
   __A		:     str												=		[4, 1_07, 38, 2_01, 1_74, 38, 2_01, 1_75, 38, 2_01, 1_76, 38, 2_01, 1_77, 5]
   self.assertEqual(encoded["""input_ids"""]				,   lowerCamelCase       )
   # decoding
   __A		:     Optional[Any]												=		tokenizer.decode(lowerCamelCase       )
   self.assertEqual(lowerCamelCase				,   """[CLS]e è é ê ë[SEP]"""       )

   # encode/decode, but with `encode` instead of `__call__`
   self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""       )       )				,   """[CLS]e è é ê ë[SEP]"""       )
 def 		lowercase_(   self			:    Union[str, Any]       ):



   """simple docstring"""





   __A		:     Optional[Any]												=		self.perceiver_tokenizer
   __A		:     int												=		["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
   # fmt: off
   __A		:     Any												=		[4, 71, 38, 1_14, 1_17, 1_16, 1_09, 38, 1_18, 1_03, 1_20, 1_03, 1_09, 1_20, 1_03, 1_18, 1_10, 38, 1_08, 1_17, 1_20, 38, 1_21, 1_23, 1_15, 1_15, 1_03, 1_20, 1_11, 1_28, 1_03, 1_22, 1_11, 1_17, 1_16, 52, 5, 0]
   # fmt: on
   __A		:     Optional[int]												=		tokenizer(lowerCamelCase				,   padding=lowerCamelCase				,   return_tensors=lowerCamelCase       )
   self.assertIsInstance(lowerCamelCase				,   lowerCamelCase       )

   if FRAMEWORK != "jax":
     __A		:     str												=		list(batch.input_ids.numpy()[0]       )
   else:
     __A		:     Optional[Any]												=		list(batch.input_ids.tolist()[0]       )

   self.assertListEqual(lowerCamelCase				,   lowerCamelCase       )

   self.assertEqual((2, 38)				,   batch.input_ids.shape       )
   self.assertEqual((2, 38)				,   batch.attention_mask.shape       )
 def 		lowercase_(   self			:    Optional[Any]       ):



   """simple docstring"""





   __A		:     str												=		self.perceiver_tokenizer
   __A		:     Any												=		["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
   __A		:     Optional[int]												=		tokenizer(lowerCamelCase				,   padding=lowerCamelCase				,   return_tensors=lowerCamelCase       )
   # check if input_ids are returned and no decoder_input_ids
   self.assertIn("""input_ids"""				,   lowerCamelCase       )
   self.assertIn("""attention_mask"""				,   lowerCamelCase       )
   self.assertNotIn("""decoder_input_ids"""				,   lowerCamelCase       )
   self.assertNotIn("""decoder_attention_mask"""				,   lowerCamelCase       )
 def 		lowercase_(   self			:    Optional[Any]       ):



   """simple docstring"""





   __A		:     Optional[int]												=		self.perceiver_tokenizer
   __A		:     Optional[Any]												=		[
       """Summary of the text.""",
       """Another summary.""",
   ]
   __A		:     Union[str, Any]												=		tokenizer(
       text_target=lowerCamelCase				,   max_length=32				,   padding="""max_length"""				,   truncation=lowerCamelCase				,   return_tensors=lowerCamelCase       )
   self.assertEqual(32				,   targets["""input_ids"""].shape[1]       )
 def 		lowercase_(   self			:    Optional[Any]       ):



   """simple docstring"""





   __A		:     int												=		self.get_tokenizers()
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"       ):
       self.assertNotEqual(tokenizer.model_max_length				,   42       )

        # Now let's start the test
   __A		:     Optional[int]												=		self.get_tokenizers()
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"       ):
       # Isolate this from the other tests because we save additional tokens/etc
       __A		:     Optional[int]												=		tempfile.mkdtemp()

       __A		:     List[str]												=		""" He is very happy, UNwant\u00E9d,running"""
       __A		:     str												=		tokenizer.encode(lowerCamelCase				,   add_special_tokens=lowerCamelCase       )
       tokenizer.save_pretrained(lowerCamelCase       )

       __A		:     Optional[int]												=		tokenizer.__class__.from_pretrained(lowerCamelCase       )
       __A		:     Optional[int]												=		after_tokenizer.encode(lowerCamelCase				,   add_special_tokens=lowerCamelCase       )
       self.assertListEqual(lowerCamelCase				,   lowerCamelCase       )

       shutil.rmtree(lowerCamelCase       )

   __A		:     Optional[Any]												=		self.get_tokenizers(model_max_length=42       )
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"       ):
       # Isolate this from the other tests because we save additional tokens/etc
       __A		:     List[Any]												=		tempfile.mkdtemp()

       __A		:     Optional[Any]												=		""" He is very happy, UNwant\u00E9d,running"""
       tokenizer.add_tokens(["""bim""", """bambam"""]       )
       __A		:     str												=		tokenizer.additional_special_tokens
       additional_special_tokens.append("""new_additional_special_token"""       )
       tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}       )
       __A		:     List[Any]												=		tokenizer.encode(lowerCamelCase				,   add_special_tokens=lowerCamelCase       )
       tokenizer.save_pretrained(lowerCamelCase       )

       __A		:     Tuple												=		tokenizer.__class__.from_pretrained(lowerCamelCase       )
       __A		:     Union[str, Any]												=		after_tokenizer.encode(lowerCamelCase				,   add_special_tokens=lowerCamelCase       )
       self.assertListEqual(lowerCamelCase				,   lowerCamelCase       )
       self.assertIn("""new_additional_special_token"""				,   after_tokenizer.additional_special_tokens       )
       self.assertEqual(after_tokenizer.model_max_length				,   42       )

       __A		:     List[str]												=		tokenizer.__class__.from_pretrained(lowerCamelCase				,   model_max_length=43       )
       self.assertEqual(tokenizer.model_max_length				,   43       )

       shutil.rmtree(lowerCamelCase       )
 def 		lowercase_(   self			:    int       ):



   """simple docstring"""





   __A		:     str												=		[]
   if self.test_slow_tokenizer:
     tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

   if self.test_rust_tokenizer:
     tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

   for tokenizer_class, tokenizer_utils in tokenizer_list:
     with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer_utils.save_pretrained(lowerCamelCase       )

       with open(os.path.join(lowerCamelCase				,   """special_tokens_map.json"""       )				,   encoding="""utf-8"""       ) as json_file:
         __A		:     int												=		json.load(lowerCamelCase       )

       with open(os.path.join(lowerCamelCase				,   """tokenizer_config.json"""       )				,   encoding="""utf-8"""       ) as json_file:
         __A		:     Any												=		json.load(lowerCamelCase       )

       __A		:     str												=		[f"<extra_id_{i}>" for i in range(1_25       )]

       __A		:     int												=		added_tokens_extra_ids + [
           """an_additional_special_token"""
       ]
       __A		:     Any												=		added_tokens_extra_ids + [
           """an_additional_special_token"""
       ]

       with open(os.path.join(lowerCamelCase				,   """special_tokens_map.json"""       )				,   """w"""				,   encoding="""utf-8"""       ) as outfile:
         json.dump(lowerCamelCase				,   lowerCamelCase       )
       with open(os.path.join(lowerCamelCase				,   """tokenizer_config.json"""       )				,   """w"""				,   encoding="""utf-8"""       ) as outfile:
         json.dump(lowerCamelCase				,   lowerCamelCase       )

       # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
       # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
       # "special_tokens_map.json" files
       __A		:     int												=		tokenizer_class.from_pretrained(
           lowerCamelCase				,   )
       self.assertIn(
           """an_additional_special_token"""				,   tokenizer_without_change_in_init.additional_special_tokens       )
       self.assertEqual(
           ["""an_additional_special_token"""]				,   tokenizer_without_change_in_init.convert_ids_to_tokens(
               tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]       )       )				,   )

       # Now we test that we can change the value of additional_special_tokens in the from_pretrained
       __A		:     int												=		added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""				,   lstrip=lowerCamelCase       )]
       __A		:     str												=		tokenizer_class.from_pretrained(
           lowerCamelCase				,   additional_special_tokens=lowerCamelCase				,   )

       self.assertIn("""a_new_additional_special_token"""				,   tokenizer.additional_special_tokens       )
       self.assertEqual(
           ["""a_new_additional_special_token"""]				,   tokenizer.convert_ids_to_tokens(
               tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]       )       )				,   )
 def 		lowercase_(   self			:    Union[str, Any]       ):



   """simple docstring"""





   __A		:     Dict												=		self.perceiver_tokenizer
   self.assertEqual(tokenizer.decode([1_78]       )				,   """�"""       )
 def 		lowercase_(   self			:    Optional[int]       ):



   """simple docstring"""





   pass
 def 		lowercase_(   self			:    Optional[Any]       ):



   """simple docstring"""





   pass
 def 		lowercase_(   self			:    Any       ):



   """simple docstring"""





   pass
 def 		lowercase_(   self			:    Union[str, Any]       ):



   """simple docstring"""





   pass






 def 		lowercase_(   self			:    Optional[int]       ):



   """simple docstring"""





   __A		:     Optional[int]												=		self.get_tokenizers(fast=lowerCamelCase				,   do_lower_case=lowerCamelCase       )
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"       ):
       __A		:     Optional[int]												=		["""[CLS]""", """t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """s""", """t""", """[SEP]"""]
       __A		:     List[Any]												=		tokenizer.convert_tokens_to_string(lowerCamelCase       )

       self.assertIsInstance(lowerCamelCase				,   lowerCamelCase       )



