



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class   snake_case_ (__a , unittest.TestCase  ):




       """simple docstring"""






       _lowerCamelCase			    =				CodeGenTokenizer
       _lowerCamelCase			    =				CodeGenTokenizerFast
       _lowerCamelCase			    =				True
       _lowerCamelCase			    =				{"""add_prefix_space""": True}
       _lowerCamelCase			    =				False



       def  A_   (     self):

             """simple docstring"""





             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             UpperCAmelCase_  :	List[Any]				      =		[
                 "l",
                 "o",
                 "w",
                 "e",
                 "r",
                 "s",
                 "t",
                 "i",
                 "d",
                 "n",
                 "\u0120",
                 "\u0120l",
                 "\u0120n",
                 "\u0120lo",
                 "\u0120low",
                 "er",
                 "\u0120lowest",
                 "\u0120newer",
                 "\u0120wider",
                 "<unk>",
                 "<|endoftext|>",
             ]
             UpperCAmelCase_  :	List[str]				      =		dict(zip(A__     ,range(len(A__))))
             UpperCAmelCase_  :	str				      =		["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
             UpperCAmelCase_  :	Any				      =		{"unk_token": "<unk>"}

             UpperCAmelCase_  :	Union[str, Any]				      =		os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES["vocab_file"])
             UpperCAmelCase_  :	int				      =		os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES["merges_file"])
             with open(self.vocab_file     ,"w"     ,encoding="utf-8") as fp:
                   fp.write(json.dumps(A__) + "\n")
             with open(self.merges_file     ,"w"     ,encoding="utf-8") as fp:
                   fp.write("\n".join(A__))



       def  A_   (     self     ,**lowercase):

             """simple docstring"""





             kwargs.update(self.special_tokens_map)
             return CodeGenTokenizer.from_pretrained(self.tmpdirname     ,**A__)



       def  A_   (     self     ,**lowercase):

             """simple docstring"""





             kwargs.update(self.special_tokens_map)
             return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     ,**A__)



       def  A_   (     self     ,lowercase):

             """simple docstring"""





             UpperCAmelCase_  :	Any				      =		"lower newer"
             UpperCAmelCase_  :	Dict				      =		"lower newer"
             return input_text, output_text



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	str				      =		CodeGenTokenizer(self.vocab_file     ,self.merges_file     ,**self.special_tokens_map)
             UpperCAmelCase_  :	int				      =		"lower newer"
             UpperCAmelCase_  :	int				      =		["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
             UpperCAmelCase_  :	List[str]				      =		tokenizer.tokenize(A__     ,add_prefix_space=A__)
             self.assertListEqual(A__     ,A__)

             UpperCAmelCase_  :	List[Any]				      =		tokens + [tokenizer.unk_token]
             UpperCAmelCase_  :	List[Any]				      =		[14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(A__)     ,A__)



       def  A_   (     self):

             """simple docstring"""





             if not self.test_rust_tokenizer:
                   return

             UpperCAmelCase_  :	Optional[Any]				      =		self.get_tokenizer()
             UpperCAmelCase_  :	Dict				      =		self.get_rust_tokenizer(add_prefix_space=A__)

             UpperCAmelCase_  :	Dict				      =		"lower newer"

             # Testing tokenization
             UpperCAmelCase_  :	Any				      =		tokenizer.tokenize(A__     ,add_prefix_space=A__)
             UpperCAmelCase_  :	Optional[int]				      =		rust_tokenizer.tokenize(A__)
             self.assertListEqual(A__     ,A__)

             # Testing conversion to ids without special tokens
             UpperCAmelCase_  :	Tuple				      =		tokenizer.encode(A__     ,add_special_tokens=A__     ,add_prefix_space=A__)
             UpperCAmelCase_  :	int				      =		rust_tokenizer.encode(A__     ,add_special_tokens=A__)
             self.assertListEqual(A__     ,A__)

             # Testing conversion to ids with special tokens
             UpperCAmelCase_  :	List[Any]				      =		self.get_rust_tokenizer(add_prefix_space=A__)
             UpperCAmelCase_  :	str				      =		tokenizer.encode(A__     ,add_prefix_space=A__)
             UpperCAmelCase_  :	Tuple				      =		rust_tokenizer.encode(A__)
             self.assertListEqual(A__     ,A__)

             # Testing the unknown token
             UpperCAmelCase_  :	Tuple				      =		tokens + [rust_tokenizer.unk_token]
             UpperCAmelCase_  :	str				      =		[14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(A__)     ,A__)



       def  A_   (     self     ,*lowercase     ,**lowercase):

             """simple docstring"""





             pass



       def  A_   (     self     ,lowercase=15):

             """simple docstring"""





             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                         UpperCAmelCase_  :	Union[str, Any]				      =		self.rust_tokenizer_class.from_pretrained(A__     ,**A__)

                         # Simple input
                         UpperCAmelCase_  :	List[Any]				      =		"This is a simple input"
                         UpperCAmelCase_  :	str				      =		["This is a simple input 1", "This is a simple input 2"]
                         UpperCAmelCase_  :	int				      =		("This is a simple input", "This is a pair")
                         UpperCAmelCase_  :	Union[str, Any]				      =		[
                             ("This is a simple input 1", "This is a simple input 2"),
                             ("This is a simple pair 1", "This is a simple pair 2"),
                         ]

                         # Simple input tests
                         self.assertRaises(A__     ,tokenizer_r.encode     ,A__     ,max_length=A__     ,padding="max_length")

                         # Simple input
                         self.assertRaises(A__     ,tokenizer_r.encode_plus     ,A__     ,max_length=A__     ,padding="max_length")

                         # Simple input
                         self.assertRaises(
                             A__     ,tokenizer_r.batch_encode_plus     ,A__     ,max_length=A__     ,padding="max_length"     ,)

                         # Pair input
                         self.assertRaises(A__     ,tokenizer_r.encode     ,A__     ,max_length=A__     ,padding="max_length")

                         # Pair input
                         self.assertRaises(A__     ,tokenizer_r.encode_plus     ,A__     ,max_length=A__     ,padding="max_length")

                         # Pair input
                         self.assertRaises(
                             A__     ,tokenizer_r.batch_encode_plus     ,A__     ,max_length=A__     ,padding="max_length"     ,)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	List[Any]				      =		CodeGenTokenizer.from_pretrained(self.tmpdirname     ,pad_token="<pad>")

             # Simple input
             UpperCAmelCase_  :	Dict				      =		"This is a simple input"
             UpperCAmelCase_  :	Dict				      =		["This is a simple input looooooooong", "This is a simple input"]
             UpperCAmelCase_  :	Optional[int]				      =		("This is a simple input", "This is a pair")
             UpperCAmelCase_  :	Tuple				      =		[
                 ("This is a simple input loooooong", "This is a simple input"),
                 ("This is a simple pair loooooong", "This is a simple pair"),
             ]

             UpperCAmelCase_  :	Dict				      =		tokenizer.pad_token_id

             UpperCAmelCase_  :	List[Any]				      =		tokenizer(A__     ,padding="max_length"     ,max_length=30     ,return_tensors="np")
             UpperCAmelCase_  :	Dict				      =		tokenizer(A__     ,padding=A__     ,truncate=A__     ,return_tensors="np")
             UpperCAmelCase_  :	Optional[int]				      =		tokenizer(*A__     ,padding="max_length"     ,max_length=60     ,return_tensors="np")
             UpperCAmelCase_  :	Tuple				      =		tokenizer(A__     ,padding=A__     ,truncate=A__     ,return_tensors="np")

             # s
             # test single string max_length padding
             self.assertEqual(out_s["input_ids"].shape[-1]     ,30)
             self.assertTrue(pad_token_id in out_s["input_ids"])
             self.assertTrue(0 in out_s["attention_mask"])

             # s2
             # test automatic padding
             self.assertEqual(out_sa["input_ids"].shape[-1]     ,33)
             # long slice doesn't have padding
             self.assertFalse(pad_token_id in out_sa["input_ids"][0])
             self.assertFalse(0 in out_sa["attention_mask"][0])
             # short slice does have padding
             self.assertTrue(pad_token_id in out_sa["input_ids"][1])
             self.assertTrue(0 in out_sa["attention_mask"][1])

             # p
             # test single pair max_length padding
             self.assertEqual(out_p["input_ids"].shape[-1]     ,60)
             self.assertTrue(pad_token_id in out_p["input_ids"])
             self.assertTrue(0 in out_p["attention_mask"])

             # p2
             # test automatic padding pair
             self.assertEqual(out_pa["input_ids"].shape[-1]     ,52)
             # long slice pair doesn't have padding
             self.assertFalse(pad_token_id in out_pa["input_ids"][0])
             self.assertFalse(0 in out_pa["attention_mask"][0])
             # short slice pair does have padding
             self.assertTrue(pad_token_id in out_pa["input_ids"][1])
             self.assertTrue(0 in out_pa["attention_mask"][1])



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	Dict				      =		"$$$"
             UpperCAmelCase_  :	Dict				      =		CodeGenTokenizer.from_pretrained(self.tmpdirname     ,bos_token=A__     ,add_bos_token=A__)

             UpperCAmelCase_  :	Any				      =		"This is a simple input"
             UpperCAmelCase_  :	Any				      =		["This is a simple input 1", "This is a simple input 2"]

             UpperCAmelCase_  :	Optional[Any]				      =		tokenizer.bos_token_id

             UpperCAmelCase_  :	Any				      =		tokenizer(A__)
             UpperCAmelCase_  :	int				      =		tokenizer(A__)

             self.assertEqual(out_s.input_ids[0]     ,A__)
             self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

             UpperCAmelCase_  :	int				      =		tokenizer.decode(out_s.input_ids)
             UpperCAmelCase_  :	int				      =		tokenizer.batch_decode(out_sa.input_ids)

             self.assertEqual(decode_s.split()[0]     ,A__)
             self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))



       @slow
       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	int				      =		CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono")

             UpperCAmelCase_  :	Dict				      =		"\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
             UpperCAmelCase_  :	str				      =		"\nif len_a > len_b:      result = a\nelse:      result = b"

             UpperCAmelCase_  :	List[str]				      =		tokenizer.encode(A__)
             UpperCAmelCase_  :	str				      =		["^#", re.escape("<|endoftext|>"), "^\'\'\'", "^\"\"\"", "\n\n\n"]
             UpperCAmelCase_  :	Union[str, Any]				      =		tokenizer.decode(A__     ,truncate_before_pattern=A__)
             self.assertEqual(A__     ,A__)



       def  A_   (     self):

             """simple docstring"""





             pass






from __future__ import annotations

from collections import deque
from collections.abc import Iterator
from dataclasses import dataclass


@dataclass
class   snake_case_ :




       """simple docstring"""






       _lowerCamelCase			    =				42
       _lowerCamelCase			    =				42


class   snake_case_ :




       """simple docstring"""






       def __init__(     self     ,lowercase):

             """simple docstring"""





             UpperCAmelCase_  :	list[list[Edge]]				      =		[[] for _ in range(lowercase)]
             UpperCAmelCase_  :	str				      =		size



       def __getitem__(     self     ,lowercase):

             """simple docstring"""





             return iter(self._graph[vertex])



       @property
       def  A_   (     self):

             """simple docstring"""





             return self._size



       def  A_   (     self     ,lowercase     ,lowercase     ,lowercase):

             """simple docstring"""





             if weight not in (0, 1):
                   raise ValueError("Edge weight must be either 0 or 1.")

             if to_vertex < 0 or to_vertex >= self.size:
                   raise ValueError("Vertex indexes must be in [0; size).")

             self._graph[from_vertex].append(Edge(lowercase     ,lowercase))



       def  A_   (     self     ,lowercase     ,lowercase):

             """simple docstring"""





             UpperCAmelCase_  :	int				      =		deque([start_vertex])
             UpperCAmelCase_  :	list[int | None]				      =		[None] * self.size
             UpperCAmelCase_  :	Dict				      =		0

             while queue:
                   UpperCAmelCase_  :	List[Any]				      =		queue.popleft()
                   UpperCAmelCase_  :	Any				      =		distances[current_vertex]
                   if current_distance is None:
                         continue

                   for edge in self[current_vertex]:
                         UpperCAmelCase_  :	Tuple				      =		current_distance + edge.weight
                         UpperCAmelCase_  :	Optional[Any]				      =		distances[edge.destination_vertex]
                         if (
                             isinstance(lowercase     ,lowercase)
                             and new_distance >= dest_vertex_distance
                         ):
                               continue
                         UpperCAmelCase_  :	Tuple				      =		new_distance
                         if edge.weight == 0:
                               queue.appendleft(edge.destination_vertex)
                         else:
                               queue.append(edge.destination_vertex)

             if distances[finish_vertex] is None:
                   raise ValueError("No path from start_vertex to finish_vertex.")

             return distances[finish_vertex]


if __name__ == "__main__":
   import doctest

   doctest.testmod()
