
import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class       _snake_case   (	_UpperCAmelCase  ,						unittest.TestCase							):
 __lowerCAmelCase  :     Optional[Any]        =			GPTaTokenizer
 __lowerCAmelCase  :     List[Any]        =			GPTaTokenizerFast
 __lowerCAmelCase  :     Union[str, Any]        =			True
 __lowerCAmelCase  :     Optional[Any]        =			{'add_prefix_space': True}
 __lowerCAmelCase  :     Any        =			False


 def      lowercase__     (					self):


   '''simple docstring'''




   super().setUp()

   # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
   lowercase__  :   Optional[Any]    		=      [
       """l""",
       """o""",
       """w""",
       """e""",
       """r""",
       """s""",
       """t""",
       """i""",
       """d""",
       """n""",
       """\u0120""",
       """\u0120l""",
       """\u0120n""",
       """\u0120lo""",
       """\u0120low""",
       """er""",
       """\u0120lowest""",
       """\u0120newer""",
       """\u0120wider""",
       """<unk>""",
       """<|endoftext|>""",
   ]
   lowercase__  :   Dict    		=      dict(zip(lowercase_					,  range(len(lowercase_))))
   lowercase__  :   List[Any]    		=      ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
   lowercase__  :   Tuple    		=      {"""unk_token""": """<unk>"""}

   lowercase__  :   Dict    		=      os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES["""vocab_file"""])
   lowercase__  :   Any    		=      os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES["""merges_file"""])
   with open(self.vocab_file					,  """w"""					,  encoding="""utf-8""") as fp:
     fp.write(json.dumps(lowercase_) + """\n""")
   with open(self.merges_file					,  """w"""					,  encoding="""utf-8""") as fp:
     fp.write("""\n""".join(lowercase_))


 def      lowercase__     (					self					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   kwargs.update(self.special_tokens_map)
   return GPTaTokenizer.from_pretrained(self.tmpdirname					,  **lowercase_)


 def      lowercase__     (					self					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   kwargs.update(self.special_tokens_map)
   return GPTaTokenizerFast.from_pretrained(self.tmpdirname					,  **lowercase_)


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__  :   Dict    		=      """lower newer"""
   lowercase__  :   int    		=      """lower newer"""
   return input_text, output_text


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   List[str]    		=      GPTaTokenizer(self.vocab_file					,  self.merges_file					,  **self.special_tokens_map)
   lowercase__  :   Union[str, Any]    		=      """lower newer"""
   lowercase__  :   List[str]    		=      ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
   lowercase__  :   Optional[int]    		=      tokenizer.tokenize(lowercase_					,  add_prefix_space=lowercase_)
   self.assertListEqual(lowercase_					,  lowercase_)

   lowercase__  :   Optional[Any]    		=      tokens + [tokenizer.unk_token]
   lowercase__  :   Dict    		=      [14, 15, 10, 9, 3, 2, 15, 19]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_)					,  lowercase_)


 def      lowercase__     (					self):


   '''simple docstring'''




   if not self.test_rust_tokenizer:
     return

   lowercase__  :   Tuple    		=      self.get_tokenizer()
   lowercase__  :   List[str]    		=      self.get_rust_tokenizer(add_prefix_space=lowercase_)

   lowercase__  :   List[str]    		=      """lower newer"""

   # Testing tokenization
   lowercase__  :   Dict    		=      tokenizer.tokenize(lowercase_					,  add_prefix_space=lowercase_)
   lowercase__  :   int    		=      rust_tokenizer.tokenize(lowercase_)
   self.assertListEqual(lowercase_					,  lowercase_)

   # Testing conversion to ids without special tokens
   lowercase__  :   Tuple    		=      tokenizer.encode(lowercase_					,  add_special_tokens=lowercase_					,  add_prefix_space=lowercase_)
   lowercase__  :   Optional[Any]    		=      rust_tokenizer.encode(lowercase_					,  add_special_tokens=lowercase_)
   self.assertListEqual(lowercase_					,  lowercase_)

   # Testing conversion to ids with special tokens
   lowercase__  :   Dict    		=      self.get_rust_tokenizer(add_prefix_space=lowercase_)
   lowercase__  :   List[str]    		=      tokenizer.encode(lowercase_					,  add_prefix_space=lowercase_)
   lowercase__  :   Dict    		=      rust_tokenizer.encode(lowercase_)
   self.assertListEqual(lowercase_					,  lowercase_)

   # Testing the unknown token
   lowercase__  :   str    		=      tokens + [rust_tokenizer.unk_token]
   lowercase__  :   Any    		=      [14, 15, 10, 9, 3, 2, 15, 19]
   self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowercase_)					,  lowercase_)


 def      lowercase__     (					self					,  *SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   pass


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_=15):


   '''simple docstring'''




   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
       lowercase__  :   List[str]    		=      self.rust_tokenizer_class.from_pretrained(lowercase_					,  **lowercase_)

       # Simple input
       lowercase__  :   Dict    		=      """This is a simple input"""
       lowercase__  :   Union[str, Any]    		=      ["""This is a simple input 1""", """This is a simple input 2"""]
       lowercase__  :   Tuple    		=      ("""This is a simple input""", """This is a pair""")
       lowercase__  :   Any    		=      [
           ("""This is a simple input 1""", """This is a simple input 2"""),
           ("""This is a simple pair 1""", """This is a simple pair 2"""),
       ]

       # Simple input tests
       self.assertRaises(lowercase_					,  tokenizer_r.encode					,  lowercase_					,  max_length=lowercase_					,  padding="""max_length""")

       # Simple input
       self.assertRaises(lowercase_					,  tokenizer_r.encode_plus					,  lowercase_					,  max_length=lowercase_					,  padding="""max_length""")

       # Simple input
       self.assertRaises(
           lowercase_					,  tokenizer_r.batch_encode_plus					,  lowercase_					,  max_length=lowercase_					,  padding="""max_length"""					,  )

       # Pair input
       self.assertRaises(lowercase_					,  tokenizer_r.encode					,  lowercase_					,  max_length=lowercase_					,  padding="""max_length""")

       # Pair input
       self.assertRaises(lowercase_					,  tokenizer_r.encode_plus					,  lowercase_					,  max_length=lowercase_					,  padding="""max_length""")

       # Pair input
       self.assertRaises(
           lowercase_					,  tokenizer_r.batch_encode_plus					,  lowercase_					,  max_length=lowercase_					,  padding="""max_length"""					,  )


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Optional[int]    		=      GPTaTokenizer.from_pretrained(self.tmpdirname					,  pad_token="""<pad>""")

   # Simple input
   lowercase__  :   str    		=      """This is a simple input"""
   lowercase__  :   str    		=      ["""This is a simple input looooooooong""", """This is a simple input"""]
   lowercase__  :   Optional[int]    		=      ("""This is a simple input""", """This is a pair""")
   lowercase__  :   Dict    		=      [
       ("""This is a simple input loooooong""", """This is a simple input"""),
       ("""This is a simple pair loooooong""", """This is a simple pair"""),
   ]

   lowercase__  :   int    		=      tokenizer.pad_token_id

   lowercase__  :   Any    		=      tokenizer(lowercase_					,  padding="""max_length"""					,  max_length=30					,  return_tensors="""np""")
   lowercase__  :   int    		=      tokenizer(lowercase_					,  padding=lowercase_					,  truncate=lowercase_					,  return_tensors="""np""")
   lowercase__  :   str    		=      tokenizer(*lowercase_					,  padding="""max_length"""					,  max_length=60					,  return_tensors="""np""")
   lowercase__  :   Optional[Any]    		=      tokenizer(lowercase_					,  padding=lowercase_					,  truncate=lowercase_					,  return_tensors="""np""")

   # s
   # test single string max_length padding
   self.assertEqual(out_s["""input_ids"""].shape[-1]					,  30)
   self.assertTrue(pad_token_id in out_s["""input_ids"""])
   self.assertTrue(0 in out_s["""attention_mask"""])

   # s2
   # test automatic padding
   self.assertEqual(out_sa["""input_ids"""].shape[-1]					,  33)
   # long slice doesn't have padding
   self.assertFalse(pad_token_id in out_sa["""input_ids"""][0])
   self.assertFalse(0 in out_sa["""attention_mask"""][0])
   # short slice does have padding
   self.assertTrue(pad_token_id in out_sa["""input_ids"""][1])
   self.assertTrue(0 in out_sa["""attention_mask"""][1])

   # p
   # test single pair max_length padding
   self.assertEqual(out_p["""input_ids"""].shape[-1]					,  60)
   self.assertTrue(pad_token_id in out_p["""input_ids"""])
   self.assertTrue(0 in out_p["""attention_mask"""])

   # p2
   # test automatic padding pair
   self.assertEqual(out_pa["""input_ids"""].shape[-1]					,  52)
   # long slice pair doesn't have padding
   self.assertFalse(pad_token_id in out_pa["""input_ids"""][0])
   self.assertFalse(0 in out_pa["""attention_mask"""][0])
   # short slice pair does have padding
   self.assertTrue(pad_token_id in out_pa["""input_ids"""][1])
   self.assertTrue(0 in out_pa["""attention_mask"""][1])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Tuple    		=      """$$$"""
   lowercase__  :   Tuple    		=      GPTaTokenizer.from_pretrained(self.tmpdirname					,  bos_token=lowercase_					,  add_bos_token=lowercase_)

   lowercase__  :   List[Any]    		=      """This is a simple input"""
   lowercase__  :   str    		=      ["""This is a simple input 1""", """This is a simple input 2"""]

   lowercase__  :   int    		=      tokenizer.bos_token_id

   lowercase__  :   Dict    		=      tokenizer(lowercase_)
   lowercase__  :   Tuple    		=      tokenizer(lowercase_)

   self.assertEqual(out_s.input_ids[0]					,  lowercase_)
   self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

   lowercase__  :   List[Any]    		=      tokenizer.decode(out_s.input_ids)
   lowercase__  :   Dict    		=      tokenizer.batch_decode(out_sa.input_ids)

   self.assertEqual(decode_s.split()[0]					,  lowercase_)
   self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))


 def      lowercase__     (					self):


   '''simple docstring'''




   pass



 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Tuple    		=      [self.get_tokenizer(do_lower_case=lowercase_					,  add_bos_token=lowercase_)]
   for tokenizer in tokenizers:
     with self.subTest(f'{tokenizer.__class__.__name__}'):
       lowercase__  :   int    		=      """Encode this."""
       lowercase__  :   Tuple    		=      """This one too please."""
       lowercase__  :   List[str]    		=      tokenizer.encode(lowercase_					,  add_special_tokens=lowercase_)
       encoded_sequence += tokenizer.encode(lowercase_					,  add_special_tokens=lowercase_)
       lowercase__  :   List[Any]    		=      tokenizer.encode_plus(
           lowercase_					,  lowercase_					,  add_special_tokens=lowercase_					,  return_special_tokens_mask=lowercase_					,  )
       lowercase__  :   int    		=      encoded_sequence_dict["""input_ids"""]
       lowercase__  :   List[str]    		=      encoded_sequence_dict["""special_tokens_mask"""]
       self.assertEqual(len(lowercase_)					,  len(lowercase_))

       lowercase__  :   int    		=      [
           (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowercase_)
       ]
       lowercase__  :   List[str]    		=      [x for x in filtered_sequence if x is not None]
       self.assertEqual(lowercase_					,  lowercase_)



@require_tokenizers
class       _snake_case   (	unittest.TestCase							):


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   str    		=      AutoTokenizer.from_pretrained("""facebook/opt-350m"""					,  from_slow=lowercase_)
   lowercase__  :   Optional[int]    		=      """A photo of a cat"""

   lowercase__  :   Any    		=      tokenizer.encode(
       lowercase_					,  )
   self.assertEqual(lowercase_					,  [2, 2_50, 13_45, 9, 10, 47_58])
   tokenizer.save_pretrained("""test_opt""")

   lowercase__  :   Optional[Any]    		=      AutoTokenizer.from_pretrained("""./test_opt""")
   lowercase__  :   List[Any]    		=      tokenizer.encode(
       lowercase_					,  )
   self.assertEqual(lowercase_					,  [2, 2_50, 13_45, 9, 10, 47_58])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   int    		=      AutoTokenizer.from_pretrained("""facebook/opt-350m"""					,  use_slow=lowercase_)
   lowercase__  :   List[Any]    		=      """A photo of a cat"""

   lowercase__  :   List[Any]    		=      tokenizer.encode(
       lowercase_					,  )
   # Same as above
   self.assertEqual(lowercase_					,  [2, 2_50, 13_45, 9, 10, 47_58])



 @unittest.skip("""This test is failing because of a bug in the fast tokenizer""")
 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Any    		=      AutoTokenizer.from_pretrained("""facebook/opt-350m"""					,  from_slow=lowercase_)

   lowercase__  :   Union[str, Any]    		=      """bos"""
   lowercase__  :   int    		=      tokenizer.get_vocab()["""bos"""]

   lowercase__  :   Union[str, Any]    		=      """A photo of a cat"""
   lowercase__  :   Dict    		=      tokenizer.encode(
       lowercase_					,  )
   # We changed the bos token
   self.assertEqual(lowercase_					,  [3_19_57, 2_50, 13_45, 9, 10, 47_58])
   tokenizer.save_pretrained("""./tok""")
   lowercase__  :   Optional[int]    		=      AutoTokenizer.from_pretrained("""./tok""")
   self.assertTrue(tokenizer.is_fast)
   lowercase__  :   Tuple    		=      tokenizer.encode(
       lowercase_					,  )
   self.assertEqual(lowercase_					,  [3_19_57, 2_50, 13_45, 9, 10, 47_58])






from __future__ import annotations

from collections.abc import Iterator
from typing import Any



class       _snake_case   :


 def __init__(					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__  :   Any    		=      data
   lowercase__  :   Node | None    		=      None



class       _snake_case   :


 def __init__(					self):


   '''simple docstring'''




   lowercase__  :   List[Any]    		=      None
   lowercase__  :   int    		=      None


 def __iter__(					self):


   '''simple docstring'''




   lowercase__  :   List[str]    		=      self.head
   while self.head:
     yield node.data
     lowercase__  :   str    		=      node.next
     if node == self.head:
       break


 def __len__(					self):


   '''simple docstring'''




   return sum(1 for _ in self)


 def __repr__(					self):


   '''simple docstring'''




   return "->".join(str(SCREAMING_SNAKE_CASE_) for item in iter(self))


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   self.insert_nth(len(self)					,  SCREAMING_SNAKE_CASE_)


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   self.insert_nth(0					,  SCREAMING_SNAKE_CASE_)


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if index < 0 or index > len(self):
     raise IndexError("""list index out of range.""")
   lowercase__  :   Optional[int]    		=      Node(SCREAMING_SNAKE_CASE_)
   if self.head is None:
     lowercase__  :   Union[str, Any]    		=      new_node  # first node points itself
     lowercase__  :   Dict    		=      new_node
   elif index == 0:  # insert at head
     lowercase__  :   int    		=      self.head
     lowercase__  :   int    		=      new_node
   else:
     lowercase__  :   List[str]    		=      self.head
     for _ in range(index - 1):
       lowercase__  :   Union[str, Any]    		=      temp.next
     lowercase__  :   int    		=      temp.next
     lowercase__  :   Optional[Any]    		=      new_node
     if index == len(self) - 1:  # insert at tail
       lowercase__  :   Union[str, Any]    		=      new_node


 def      lowercase__     (					self):


   '''simple docstring'''




   return self.delete_nth(0)


 def      lowercase__     (					self):


   '''simple docstring'''




   return self.delete_nth(len(self) - 1)


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_ = 0):


   '''simple docstring'''




   if not 0 <= index < len(self):
     raise IndexError("""list index out of range.""")
   lowercase__  :   str    		=      self.head
   if self.head == self.tail:  # just one node
     lowercase__  :   List[Any]    		=      None
   elif index == 0:  # delete head node
     lowercase__  :   str    		=      self.tail.next.next
     lowercase__  :   List[Any]    		=      self.head.next
   else:
     lowercase__  :   Optional[int]    		=      self.head
     for _ in range(index - 1):
       lowercase__  :   List[Any]    		=      temp.next
     lowercase__  :   Optional[Any]    		=      temp.next
     lowercase__  :   Dict    		=      temp.next.next
     if index == len(self) - 1:  # delete at tail
       lowercase__  :   Tuple    		=      temp
   return delete_node.data



 def      lowercase__     (					self):


   '''simple docstring'''




   return len(self) == 0

def 		UpperCamelCase	(      )		->	None:


  '''simple docstring'''






  lowercase__  :   int    		=      CircularLinkedList()
  assert len(lowercase_   ) == 0
  assert circular_linked_list.is_empty() is True
  assert str(lowercase_   ) == ""

  try:
    circular_linked_list.delete_front()
    raise AssertionError  # This should not happen
  except IndexError:
    assert True  # This should happen

  try:
    circular_linked_list.delete_tail()
    raise AssertionError  # This should not happen
  except IndexError:
    assert True  # This should happen

  try:
    circular_linked_list.delete_nth(-1   )
    raise AssertionError
  except IndexError:
    assert True

  try:
    circular_linked_list.delete_nth(0   )
    raise AssertionError
  except IndexError:
    assert True

  assert circular_linked_list.is_empty() is True
  for i in range(5   ):
    assert len(lowercase_   ) == i
    circular_linked_list.insert_nth(lowercase_ , i + 1   )
  assert str(lowercase_   ) == "->".join(str(lowercase_   ) for i in range(1 , 6   )   )

  circular_linked_list.insert_tail(6   )
  assert str(lowercase_   ) == "->".join(str(lowercase_   ) for i in range(1 , 7   )   )
  circular_linked_list.insert_head(0   )
  assert str(lowercase_   ) == "->".join(str(lowercase_   ) for i in range(0 , 7   )   )

  assert circular_linked_list.delete_front() == 0
  assert circular_linked_list.delete_tail() == 6
  assert str(lowercase_   ) == "->".join(str(lowercase_   ) for i in range(1 , 6   )   )
  assert circular_linked_list.delete_nth(2   ) == 3

  circular_linked_list.insert_nth(2 , 3   )
  assert str(lowercase_   ) == "->".join(str(lowercase_   ) for i in range(1 , 6   )   )

  assert circular_linked_list.is_empty() is False


if __name__ == "__main__":
     import doctest

     doctest.testmod()






