

import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
      lowerCAmelCase__       = "pt"
elif is_tf_available():
      lowerCAmelCase__       = "tf"
else:
      lowerCAmelCase__       = "jax"






class 						__magic_name__    (       _snake_case	,						unittest.TestCase    ):
 UpperCAmelCase							    =						ByTaTokenizer
 UpperCAmelCase							    =						False


 def _UpperCamelCase	(						self		: List[str]    )    ->      Any:
   super().setUp()
   UpperCAmelCase					      =  ByTaTokenizer()
   tokenizer.save_pretrained(self.tmpdirname    )


 @cached_property
 def _UpperCamelCase	(						self		: Dict    )    ->      Optional[int]:
   return ByTaTokenizer.from_pretrained("google/byt5-small"    )


 def _UpperCamelCase	(						self		: Any			,						**lowerCAmelCase__		: Union[str, Any]    )    ->      ByTaTokenizer:
   return self.tokenizer_class.from_pretrained(self.tmpdirname			,						**lowerCAmelCase__    )


 def _UpperCamelCase	(						self		: Tuple			,						lowerCAmelCase__		: Dict			,						lowerCAmelCase__		: Union[str, Any]=False			,						lowerCAmelCase__		: Optional[int]=2_0			,						lowerCAmelCase__		: str=5    )    ->      Tuple[str, list]:
   # XXX The default common tokenizer tests assume that every ID is decodable on its own.
   # This assumption is invalid for ByT5 because single bytes might not be
   # valid utf-8 (byte 128 for instance).
   # Here we're overriding the smallest possible method to provide
   # a clean sequence without making the same assumption.

   UpperCAmelCase					      =  []
   for i in range(len(lowerCAmelCase__    )    ):
     try:
       UpperCAmelCase					      =  tokenizer.decode([i]			,						clean_up_tokenization_spaces=lowerCAmelCase__    )
     except UnicodeDecodeError:
       pass
     toks.append((i, tok)    )

   UpperCAmelCase					      =  list(filter(lambda lowerCAmelCase__    : re.match(r"^[ a-zA-Z]+$"			,						t[1]    )			,						lowerCAmelCase__    )    )
   UpperCAmelCase					      =  list(filter(lambda lowerCAmelCase__    : [t[0]] == tokenizer.encode(t[1]			,						add_special_tokens=lowerCAmelCase__    )			,						lowerCAmelCase__    )    )
   if max_length is not None and len(lowerCAmelCase__    ) > max_length:
     UpperCAmelCase					      =  toks[:max_length]
   if min_length is not None and len(lowerCAmelCase__    ) < min_length and len(lowerCAmelCase__    ) > 0:
     while len(lowerCAmelCase__    ) < min_length:
       UpperCAmelCase					      =  toks + toks
        # toks_str = [t[1] for t in toks]
   UpperCAmelCase					      =  [t[0] for t in toks]

   # Ensure consistency
   UpperCAmelCase					      =  tokenizer.decode(lowerCAmelCase__			,						clean_up_tokenization_spaces=lowerCAmelCase__    )
   if " " not in output_txt and len(lowerCAmelCase__    ) > 1:
     UpperCAmelCase					      =  (
         tokenizer.decode([toks_ids[0]]			,						clean_up_tokenization_spaces=lowerCAmelCase__    )
         + " "
         + tokenizer.decode(toks_ids[1:]			,						clean_up_tokenization_spaces=lowerCAmelCase__    )
     )
   if with_prefix_space:
     UpperCAmelCase					      =  " " + output_txt
   UpperCAmelCase					      =  tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__    )
   return output_txt, output_ids


 def _UpperCamelCase	(						self		: List[Any]    )    ->      List[str]:
   UpperCAmelCase					      =  self.ta_base_tokenizer
   UpperCAmelCase					      =  tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]    )
   UpperCAmelCase					      =  tokenizer(["hi", "I went to the gym", ""]    )
   self.assertListEqual(batch_with_eos_added["input_ids"]			,						batch_without_eos_added["input_ids"]    )


 def _UpperCamelCase	(						self		: int    )    ->      List[Any]:
   UpperCAmelCase					      =  self.ta_base_tokenizer
   UpperCAmelCase					      =  "Unicode €."
   UpperCAmelCase					      =  tokenizer(lowerCAmelCase__    )
   UpperCAmelCase					      =  [8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
   self.assertEqual(encoded["input_ids"]			,						lowerCAmelCase__    )

   # decoding
   UpperCAmelCase					      =  tokenizer.decode(lowerCAmelCase__    )
   self.assertEqual(lowerCAmelCase__			,						"Unicode €.</s>"    )

   UpperCAmelCase					      =  tokenizer("e è é ê ë"    )
   UpperCAmelCase					      =  [1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
   self.assertEqual(encoded["input_ids"]			,						lowerCAmelCase__    )
   # decoding
   UpperCAmelCase					      =  tokenizer.decode(lowerCAmelCase__    )
   self.assertEqual(lowerCAmelCase__			,						"e è é ê ë</s>"    )

   # encode/decode, but with `encode` instead of `__call__`
   self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"    )    )			,						"e è é ê ë</s>"    )


 def _UpperCamelCase	(						self		: Optional[Any]    )    ->      Optional[Any]:
   UpperCAmelCase					      =  self.ta_base_tokenizer
   UpperCAmelCase					      =  ["A long paragraph for summarization.", "Another paragraph for summarization."]
   # fmt: off
   UpperCAmelCase					      =  [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
   # fmt: on
   UpperCAmelCase					      =  tokenizer(lowerCAmelCase__			,						padding=lowerCAmelCase__			,						return_tensors=lowerCAmelCase__    )
   self.assertIsInstance(lowerCAmelCase__			,						lowerCAmelCase__    )

   if FRAMEWORK != "jax":
     UpperCAmelCase					      =  list(batch.input_ids.numpy()[0]    )
   else:
     UpperCAmelCase					      =  list(batch.input_ids.tolist()[0]    )

   self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__    )

   self.assertEqual((2, 3_7)			,						batch.input_ids.shape    )
   self.assertEqual((2, 3_7)			,						batch.attention_mask.shape    )


 def _UpperCamelCase	(						self		: Optional[int]    )    ->      Optional[int]:
   UpperCAmelCase					      =  self.ta_base_tokenizer
   UpperCAmelCase					      =  ["A long paragraph for summarization.", "Another paragraph for summarization."]
   UpperCAmelCase					      =  tokenizer(lowerCAmelCase__			,						padding=lowerCAmelCase__			,						return_tensors=lowerCAmelCase__    )
   # check if input_ids are returned and no decoder_input_ids
   self.assertIn("input_ids"			,						lowerCAmelCase__    )
   self.assertIn("attention_mask"			,						lowerCAmelCase__    )
   self.assertNotIn("decoder_input_ids"			,						lowerCAmelCase__    )
   self.assertNotIn("decoder_attention_mask"			,						lowerCAmelCase__    )


 def _UpperCamelCase	(						self		: Tuple    )    ->      List[Any]:
   UpperCAmelCase					      =  self.ta_base_tokenizer
   UpperCAmelCase					      =  [
       "Summary of the text.",
       "Another summary.",
   ]
   UpperCAmelCase					      =  tokenizer(
       text_target=lowerCAmelCase__			,						max_length=3_2			,						padding="max_length"			,						truncation=lowerCAmelCase__			,						return_tensors=lowerCAmelCase__    )
   self.assertEqual(3_2			,						targets["input_ids"].shape[1]    )


 def _UpperCamelCase	(						self		: Optional[Any]    )    ->      List[Any]:
   UpperCAmelCase					      =  self.ta_base_tokenizer
   UpperCAmelCase					      =  ["A long paragraph for summarization. </s>"]
   UpperCAmelCase					      =  ["Summary of the text. </s>"]
   # fmt: off
   UpperCAmelCase					      =  [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
   UpperCAmelCase					      =  [8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
   # fmt: on

   UpperCAmelCase					      =  tokenizer(lowerCAmelCase__			,						text_target=lowerCAmelCase__    )

   self.assertEqual(lowerCAmelCase__			,						batch["input_ids"][0]    )
   self.assertEqual(lowerCAmelCase__			,						batch["labels"][0]    )


 def _UpperCamelCase	(						self		: List[str]    )    ->      int:
   # safety check on max_len default value so we are sure the test works
   UpperCAmelCase					      =  self.get_tokenizers()
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"    ):
       self.assertNotEqual(tokenizer.model_max_length			,						4_2    )

        # Now let's start the test
   UpperCAmelCase					      =  self.get_tokenizers()
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"    ):
       # Isolate this from the other tests because we save additional tokens/etc
       UpperCAmelCase					      =  tempfile.mkdtemp()

       UpperCAmelCase					      =  " He is very happy, UNwant\u00E9d,running"
       UpperCAmelCase					      =  tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__    )
       tokenizer.save_pretrained(lowerCAmelCase__    )

       UpperCAmelCase					      =  tokenizer.__class__.from_pretrained(lowerCAmelCase__    )
       UpperCAmelCase					      =  after_tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__    )
       self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__    )

       shutil.rmtree(lowerCAmelCase__    )

   UpperCAmelCase					      =  self.get_tokenizers(model_max_length=4_2    )
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"    ):
       # Isolate this from the other tests because we save additional tokens/etc
       UpperCAmelCase					      =  tempfile.mkdtemp()

       UpperCAmelCase					      =  " He is very happy, UNwant\u00E9d,running"
       tokenizer.add_tokens(["bim", "bambam"]    )
       UpperCAmelCase					      =  tokenizer.additional_special_tokens
       additional_special_tokens.append("new_additional_special_token"    )
       tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}    )
       UpperCAmelCase					      =  tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__    )
       tokenizer.save_pretrained(lowerCAmelCase__    )

       UpperCAmelCase					      =  tokenizer.__class__.from_pretrained(lowerCAmelCase__    )
       UpperCAmelCase					      =  after_tokenizer.encode(lowerCAmelCase__			,						add_special_tokens=lowerCAmelCase__    )
       self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__    )
       self.assertIn("new_additional_special_token"			,						after_tokenizer.additional_special_tokens    )
       self.assertEqual(after_tokenizer.model_max_length			,						4_2    )

       UpperCAmelCase					      =  tokenizer.__class__.from_pretrained(lowerCAmelCase__			,						model_max_length=4_3    )
       self.assertEqual(tokenizer.model_max_length			,						4_3    )

       shutil.rmtree(lowerCAmelCase__    )


 def _UpperCamelCase	(						self		: Any    )    ->      int:
   UpperCAmelCase					      =  []
   if self.test_slow_tokenizer:
     tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

   if self.test_rust_tokenizer:
     tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

   for tokenizer_class, tokenizer_utils in tokenizer_list:
     with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer_utils.save_pretrained(lowerCAmelCase__    )

       with open(os.path.join(lowerCAmelCase__			,						"special_tokens_map.json"    )			,						encoding="utf-8"    ) as json_file:
         UpperCAmelCase					      =  json.load(lowerCAmelCase__    )

       with open(os.path.join(lowerCAmelCase__			,						"tokenizer_config.json"    )			,						encoding="utf-8"    ) as json_file:
         UpperCAmelCase					      =  json.load(lowerCAmelCase__    )

       UpperCAmelCase					      =  [f"<extra_id_{i}>" for i in range(1_2_5    )]

       UpperCAmelCase					      =  added_tokens_extra_ids + [
           "an_additional_special_token"
       ]
       UpperCAmelCase					      =  added_tokens_extra_ids + [
           "an_additional_special_token"
       ]

       with open(os.path.join(lowerCAmelCase__			,						"special_tokens_map.json"    )			,						"w"			,						encoding="utf-8"    ) as outfile:
         json.dump(lowerCAmelCase__			,						lowerCAmelCase__    )
       with open(os.path.join(lowerCAmelCase__			,						"tokenizer_config.json"    )			,						"w"			,						encoding="utf-8"    ) as outfile:
         json.dump(lowerCAmelCase__			,						lowerCAmelCase__    )

       # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
       # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
       # "special_tokens_map.json" files
       UpperCAmelCase					      =  tokenizer_class.from_pretrained(
           lowerCAmelCase__			,						)
       self.assertIn(
           "an_additional_special_token"			,						tokenizer_without_change_in_init.additional_special_tokens    )
       # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
       self.assertEqual(
           ["an_additional_special_token"]			,						tokenizer_without_change_in_init.convert_ids_to_tokens(
               tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]    )    )			,						)

       # Now we test that we can change the value of additional_special_tokens in the from_pretrained
       UpperCAmelCase					      =  added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"			,						lstrip=lowerCAmelCase__    )]
       UpperCAmelCase					      =  tokenizer_class.from_pretrained(
           lowerCAmelCase__			,						additional_special_tokens=lowerCAmelCase__			,						)

       self.assertIn("a_new_additional_special_token"			,						tokenizer.additional_special_tokens    )
       self.assertEqual(
           ["a_new_additional_special_token"]			,						tokenizer.convert_ids_to_tokens(
               tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]    )    )			,						)


 def _UpperCamelCase	(						self		: str    )    ->      Tuple:
   UpperCAmelCase					      =  []
   if self.test_slow_tokenizer:
     tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

   if self.test_rust_tokenizer:
     tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

   for tokenizer_class, tokenizer_utils in tokenizer_list:
     with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer_utils.save_pretrained(lowerCAmelCase__    )

       UpperCAmelCase					      =  tokenizer_class.from_pretrained(lowerCAmelCase__    )

       self.assertTrue(tokenizer.decode([2_5_5]    ) == ""    )


 def _UpperCamelCase	(						self		: Optional[Any]    )    ->      Dict:
   pass


 def _UpperCamelCase	(						self		: Optional[Any]    )    ->      Tuple:
   pass


 def _UpperCamelCase	(						self		: Dict    )    ->      List[str]:
   pass


 def _UpperCamelCase	(						self		: Optional[int]    )    ->      int:
   pass


 def _UpperCamelCase	(						self		: int    )    ->      int:
   # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
   # and special added tokens as tokens
   UpperCAmelCase					      =  self.get_tokenizers(fast=lowerCAmelCase__			,						do_lower_case=lowerCAmelCase__    )
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"    ):
       UpperCAmelCase					      =  ["t", "h", "i", "s", " ", "i", "s", " ", "a", " ", "t", "e", "x", "t", "</s>"]
       UpperCAmelCase					      =  tokenizer.convert_tokens_to_string(lowerCAmelCase__    )

       self.assertIsInstance(lowerCAmelCase__			,						lowerCAmelCase__    )


 def _UpperCamelCase	(						self		: Optional[int]    )    ->      List[str]:
   UpperCAmelCase					      =  self.get_tokenizers()
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"    ):
       UpperCAmelCase					      =  [
           "bos_token",
           "eos_token",
           "unk_token",
           "sep_token",
           "pad_token",
           "cls_token",
           "mask_token",
       ]

       UpperCAmelCase					      =  0
       UpperCAmelCase					      =  tokenizer.convert_ids_to_tokens(
           lowerCAmelCase__			,						skip_special_tokens=lowerCAmelCase__    )

       for attr in attributes_list:
         setattr(lowerCAmelCase__			,						attr + "_id"			,						lowerCAmelCase__    )
         self.assertEqual(getattr(lowerCAmelCase__			,						lowerCAmelCase__    )			,						lowerCAmelCase__    )
         self.assertEqual(getattr(lowerCAmelCase__			,						attr + "_id"    )			,						lowerCAmelCase__    )

         setattr(lowerCAmelCase__			,						attr + "_id"			,						lowerCAmelCase__    )
         self.assertEqual(getattr(lowerCAmelCase__			,						lowerCAmelCase__    )			,						lowerCAmelCase__    )
         self.assertEqual(getattr(lowerCAmelCase__			,						attr + "_id"    )			,						lowerCAmelCase__    )

       setattr(lowerCAmelCase__			,						"additional_special_tokens_ids"			,						[]    )
       self.assertListEqual(getattr(lowerCAmelCase__			,						"additional_special_tokens"    )			,						[]    )
       self.assertListEqual(getattr(lowerCAmelCase__			,						"additional_special_tokens_ids"    )			,						[]    )

       setattr(lowerCAmelCase__			,						"additional_special_tokens_ids"			,						[token_id_to_test_setters]    )
       self.assertListEqual(getattr(lowerCAmelCase__			,						"additional_special_tokens"    )			,						[token_to_test_setters]    )
       self.assertListEqual(getattr(lowerCAmelCase__			,						"additional_special_tokens_ids"    )			,						[token_id_to_test_setters]    )






import numpy

# List of input, output pairs
lowerCAmelCase__       = (
    ((5, 2, 3), 15),
    ((6, 5, 9), 25),
    ((11, 12, 13), 41),
    ((1, 1, 1), 8),
    ((11, 12, 13), 41),
)
lowerCAmelCase__       = (((515, 22, 13), 555), ((61, 35, 49), 150))
lowerCAmelCase__       = [2, 4, 1, 5]
lowerCAmelCase__       = len(train_data)
lowerCAmelCase__       = 0.0_0_9
def      _lowerCAmelCase(	__A   ,       __A="train"			):
  return calculate_hypothesis_value(__A   ,       __A			) - output(
      __A   ,       __A			)
def      _lowerCAmelCase(	__A			):
  UpperCAmelCase					      =  0
  for i in range(len(__A			) - 1			):
    hyp_val += data_input_tuple[i] * parameter_vector[i + 1]
  hyp_val += parameter_vector[0]
  return hyp_val
def      _lowerCAmelCase(	__A   ,       __A			):
  if data_set == "train":
    return train_data[example_no][1]
  elif data_set == "test":
    return test_data[example_no][1]
  return None
def      _lowerCAmelCase(	__A   ,       __A			):
  if data_set == "train":
    return _hypothesis_value(train_data[example_no][0]			)
  elif data_set == "test":
    return _hypothesis_value(test_data[example_no][0]			)
  return None
def      _lowerCAmelCase(	__A   ,       __A=m			):
  UpperCAmelCase					      =  0
  for i in range(__A			):
    if index == -1:
      summation_value += _error(__A			)
    else:
      summation_value += _error(__A			) * train_data[i][0][index]
  return summation_value
def      _lowerCAmelCase(	__A			):
  UpperCAmelCase					      =  summation_of_cost_derivative(__A   ,       __A			) / m
  return cost_derivative_value
def      _lowerCAmelCase(	):
  global parameter_vector
  # Tune these values to set a tolerance value for predicted output
  UpperCAmelCase					      =  0.000002
  UpperCAmelCase					      =  0
  UpperCAmelCase					      =  0
  while True:
    j += 1
    UpperCAmelCase					      =  [0, 0, 0, 0]
    for i in range(0   ,       len(__A			)			):
      UpperCAmelCase					      =  get_cost_derivative(i - 1			)
      UpperCAmelCase					      =  (
          parameter_vector[i] - LEARNING_RATE * cost_derivative
      )
    if numpy.allclose(
        __A   ,       __A   ,       atol=__A   ,       rtol=__A   ,       ):
      break
    UpperCAmelCase					      =  temp_parameter_vector
  print(("Number of iterations:", j)			)
def      _lowerCAmelCase(	):
  for i in range(len(__A			)			):
    print(("Actual output value:", output(__A   ,       "test"			))			)
    print(("Hypothesis output:", calculate_hypothesis_value(__A   ,       "test"			))			)


if __name__ == "__main__":
      run_gradient_descent()
      print("\nTesting gradient descent for a linear hypothesis function.\n")
      test_gradient_descent()



