






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class  SCREAMING_SNAKE_CASE			(			lowerCAmelCase      ,		unittest.TestCase					):
     '''simple docstring'''


     UpperCamelCase_							:		List[str]			=	CodeGenTokenizer
     UpperCamelCase_							:		List[Any]			=	CodeGenTokenizerFast
     UpperCamelCase_							:		Tuple			=	True
     UpperCamelCase_							:		List[Any]			=	{'''add_prefix_space''': True}
     UpperCamelCase_							:		Any			=	False


     def 				_A       (				self       :	int      ):
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       SCREAMING_SNAKE_CASE       :      Dict							=    [
           "l",
           "o",
           "w",
           "e",
           "r",
           "s",
           "t",
           "i",
           "d",
           "n",
           "\u0120",
           "\u0120l",
           "\u0120n",
           "\u0120lo",
           "\u0120low",
           "er",
           "\u0120lowest",
           "\u0120newer",
           "\u0120wider",
           "<unk>",
           "<|endoftext|>",
       ]
       SCREAMING_SNAKE_CASE       :      Tuple							=    dict(zip(UpperCAmelCase_					, range(len(UpperCAmelCase_      )      )      )      )
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    {"unk_token": "<unk>"}

       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    os.path.join(self.tmpdirname					, VOCAB_FILES_NAMES["vocab_file"]      )
       SCREAMING_SNAKE_CASE       :      Tuple							=    os.path.join(self.tmpdirname					, VOCAB_FILES_NAMES["merges_file"]      )
       with open(self.vocab_file					, "w"					, encoding="utf-8"      ) as fp:
         fp.write(json.dumps(UpperCAmelCase_      ) + "\n"      )
       with open(self.merges_file					, "w"					, encoding="utf-8"      ) as fp:
         fp.write("\n".join(UpperCAmelCase_      )      )


     def 				_A       (				self       :	Any					, **UpperCAmelCase_       :	Optional[int]      ):
       kwargs.update(self.special_tokens_map      )
       return CodeGenTokenizer.from_pretrained(self.tmpdirname					, **UpperCAmelCase_      )


     def 				_A       (				self       :	Dict					, **UpperCAmelCase_       :	Union[str, Any]      ):
       kwargs.update(self.special_tokens_map      )
       return CodeGenTokenizerFast.from_pretrained(self.tmpdirname					, **UpperCAmelCase_      )


     def 				_A       (				self       :	List[Any]					, UpperCAmelCase_       :	List[Any]      ):
       SCREAMING_SNAKE_CASE       :      Any							=    "lower newer"
       SCREAMING_SNAKE_CASE       :      List[str]							=    "lower newer"
       return input_text, output_text


     def 				_A       (				self       :	Union[str, Any]      ):
       SCREAMING_SNAKE_CASE       :      Any							=    CodeGenTokenizer(self.vocab_file					, self.merges_file					, **self.special_tokens_map      )
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    "lower newer"
       SCREAMING_SNAKE_CASE       :      Any							=    ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    tokenizer.tokenize(UpperCAmelCase_					, add_prefix_space=UpperCAmelCase_      )
       self.assertListEqual(UpperCAmelCase_					, UpperCAmelCase_      )

       SCREAMING_SNAKE_CASE       :      Any							=    tokens + [tokenizer.unk_token]
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )					, UpperCAmelCase_      )


     def 				_A       (				self       :	Any      ):
       if not self.test_rust_tokenizer:
         return

       SCREAMING_SNAKE_CASE       :      Tuple							=    self.get_tokenizer()
       SCREAMING_SNAKE_CASE       :      Any							=    self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_      )

       SCREAMING_SNAKE_CASE       :      Dict							=    "lower newer"

       # Testing tokenization
       SCREAMING_SNAKE_CASE       :      Tuple							=    tokenizer.tokenize(UpperCAmelCase_					, add_prefix_space=UpperCAmelCase_      )
       SCREAMING_SNAKE_CASE       :      List[Any]							=    rust_tokenizer.tokenize(UpperCAmelCase_      )
       self.assertListEqual(UpperCAmelCase_					, UpperCAmelCase_      )

       # Testing conversion to ids without special tokens
       SCREAMING_SNAKE_CASE       :      List[Any]							=    tokenizer.encode(UpperCAmelCase_					, add_special_tokens=UpperCAmelCase_					, add_prefix_space=UpperCAmelCase_      )
       SCREAMING_SNAKE_CASE       :      Tuple							=    rust_tokenizer.encode(UpperCAmelCase_					, add_special_tokens=UpperCAmelCase_      )
       self.assertListEqual(UpperCAmelCase_					, UpperCAmelCase_      )

       # Testing conversion to ids with special tokens
       SCREAMING_SNAKE_CASE       :      Dict							=    self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_      )
       SCREAMING_SNAKE_CASE       :      List[str]							=    tokenizer.encode(UpperCAmelCase_					, add_prefix_space=UpperCAmelCase_      )
       SCREAMING_SNAKE_CASE       :      Dict							=    rust_tokenizer.encode(UpperCAmelCase_      )
       self.assertListEqual(UpperCAmelCase_					, UpperCAmelCase_      )

       # Testing the unknown token
       SCREAMING_SNAKE_CASE       :      str							=    tokens + [rust_tokenizer.unk_token]
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )					, UpperCAmelCase_      )


     def 				_A       (				self       :	Tuple					, *UpperCAmelCase_       :	Tuple					, **UpperCAmelCase_       :	Optional[int]      ):
       # It's very difficult to mix/test pretokenization with byte-level
       # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
       pass


     def 				_A       (				self       :	Union[str, Any]					, UpperCAmelCase_       :	Any=15      ):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
           SCREAMING_SNAKE_CASE       :      List[Any]							=    self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_					, **UpperCAmelCase_      )

           # Simple input
           SCREAMING_SNAKE_CASE       :      List[Any]							=    "This is a simple input"
           SCREAMING_SNAKE_CASE       :      List[Any]							=    ["This is a simple input 1", "This is a simple input 2"]
           SCREAMING_SNAKE_CASE       :      Optional[int]							=    ("This is a simple input", "This is a pair")
           SCREAMING_SNAKE_CASE       :      Optional[int]							=    [
               ("This is a simple input 1", "This is a simple input 2"),
               ("This is a simple pair 1", "This is a simple pair 2"),
           ]

           # Simple input tests
           self.assertRaises(UpperCAmelCase_					, tokenizer_r.encode					, UpperCAmelCase_					, max_length=UpperCAmelCase_					, padding="max_length"      )

           # Simple input
           self.assertRaises(UpperCAmelCase_					, tokenizer_r.encode_plus					, UpperCAmelCase_					, max_length=UpperCAmelCase_					, padding="max_length"      )

           # Simple input
           self.assertRaises(
               UpperCAmelCase_					, tokenizer_r.batch_encode_plus					, UpperCAmelCase_					, max_length=UpperCAmelCase_					, padding="max_length"					, )

           # Pair input
           self.assertRaises(UpperCAmelCase_					, tokenizer_r.encode					, UpperCAmelCase_					, max_length=UpperCAmelCase_					, padding="max_length"      )

           # Pair input
           self.assertRaises(UpperCAmelCase_					, tokenizer_r.encode_plus					, UpperCAmelCase_					, max_length=UpperCAmelCase_					, padding="max_length"      )

           # Pair input
           self.assertRaises(
               UpperCAmelCase_					, tokenizer_r.batch_encode_plus					, UpperCAmelCase_					, max_length=UpperCAmelCase_					, padding="max_length"					, )


     def 				_A       (				self       :	Dict      ):
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    CodeGenTokenizer.from_pretrained(self.tmpdirname					, pad_token="<pad>"      )

       # Simple input
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    "This is a simple input"
       SCREAMING_SNAKE_CASE       :      Dict							=    ["This is a simple input looooooooong", "This is a simple input"]
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    ("This is a simple input", "This is a pair")
       SCREAMING_SNAKE_CASE       :      Any							=    [
           ("This is a simple input loooooong", "This is a simple input"),
           ("This is a simple pair loooooong", "This is a simple pair"),
       ]

       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    tokenizer.pad_token_id

       SCREAMING_SNAKE_CASE       :      Dict							=    tokenizer(UpperCAmelCase_					, padding="max_length"					, max_length=30					, return_tensors="np"      )
       SCREAMING_SNAKE_CASE       :      List[str]							=    tokenizer(UpperCAmelCase_					, padding=UpperCAmelCase_					, truncate=UpperCAmelCase_					, return_tensors="np"      )
       SCREAMING_SNAKE_CASE       :      List[Any]							=    tokenizer(*UpperCAmelCase_					, padding="max_length"					, max_length=60					, return_tensors="np"      )
       SCREAMING_SNAKE_CASE       :      Any							=    tokenizer(UpperCAmelCase_					, padding=UpperCAmelCase_					, truncate=UpperCAmelCase_					, return_tensors="np"      )

       # s
       # test single string max_length padding
       self.assertEqual(out_s["input_ids"].shape[-1]					, 30      )
       self.assertTrue(pad_token_id in out_s["input_ids"]      )
       self.assertTrue(0 in out_s["attention_mask"]      )

       # s2
       # test automatic padding
       self.assertEqual(out_sa["input_ids"].shape[-1]					, 33      )
       # long slice doesn't have padding
       self.assertFalse(pad_token_id in out_sa["input_ids"][0]      )
       self.assertFalse(0 in out_sa["attention_mask"][0]      )
       # short slice does have padding
       self.assertTrue(pad_token_id in out_sa["input_ids"][1]      )
       self.assertTrue(0 in out_sa["attention_mask"][1]      )

       # p
       # test single pair max_length padding
       self.assertEqual(out_p["input_ids"].shape[-1]					, 60      )
       self.assertTrue(pad_token_id in out_p["input_ids"]      )
       self.assertTrue(0 in out_p["attention_mask"]      )

       # p2
       # test automatic padding pair
       self.assertEqual(out_pa["input_ids"].shape[-1]					, 52      )
       # long slice pair doesn't have padding
       self.assertFalse(pad_token_id in out_pa["input_ids"][0]      )
       self.assertFalse(0 in out_pa["attention_mask"][0]      )
       # short slice pair does have padding
       self.assertTrue(pad_token_id in out_pa["input_ids"][1]      )
       self.assertTrue(0 in out_pa["attention_mask"][1]      )


     def 				_A       (				self       :	List[Any]      ):
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    "$$$"
       SCREAMING_SNAKE_CASE       :      List[str]							=    CodeGenTokenizer.from_pretrained(self.tmpdirname					, bos_token=UpperCAmelCase_					, add_bos_token=UpperCAmelCase_      )

       SCREAMING_SNAKE_CASE       :      Tuple							=    "This is a simple input"
       SCREAMING_SNAKE_CASE       :      str							=    ["This is a simple input 1", "This is a simple input 2"]

       SCREAMING_SNAKE_CASE       :      List[Any]							=    tokenizer.bos_token_id

       SCREAMING_SNAKE_CASE       :      Dict							=    tokenizer(UpperCAmelCase_      )
       SCREAMING_SNAKE_CASE       :      Tuple							=    tokenizer(UpperCAmelCase_      )

       self.assertEqual(out_s.input_ids[0]					, UpperCAmelCase_      )
       self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

       SCREAMING_SNAKE_CASE       :      Optional[int]							=    tokenizer.decode(out_s.input_ids      )
       SCREAMING_SNAKE_CASE       :      int							=    tokenizer.batch_decode(out_sa.input_ids      )

       self.assertEqual(decode_s.split()[0]					, UpperCAmelCase_      )
       self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )


     @slow
     def 				_A       (				self       :	str      ):
       SCREAMING_SNAKE_CASE       :      int							=    CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"      )

       SCREAMING_SNAKE_CASE       :      Optional[int]							=    "\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
       SCREAMING_SNAKE_CASE       :      Any							=    "\nif len_a > len_b:      result = a\nelse:      result = b"

       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    tokenizer.encode(UpperCAmelCase_      )
       SCREAMING_SNAKE_CASE       :      List[Any]							=    ["^#", re.escape("<|endoftext|>"      ), "^'''", "^\"\"\"", "\n\n\n"]
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    tokenizer.decode(UpperCAmelCase_					, truncate_before_pattern=UpperCAmelCase_      )
       self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_      )






     def 				_A       (				self       :	List[Any]      ):
       pass














from ...utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_torch_available,
    is_transformers_available,
)


try:
   if not (is_transformers_available() and is_torch_available()):
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   from ...utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
   from .multicontrolnet import MultiControlNetModel
   from .pipeline_controlnet import StableDiffusionControlNetPipeline
   from .pipeline_controlnet_imgaimg import StableDiffusionControlNetImgaImgPipeline
   from .pipeline_controlnet_inpaint import StableDiffusionControlNetInpaintPipeline


if is_transformers_available() and is_flax_available():
   from .pipeline_flax_controlnet import FlaxStableDiffusionControlNetPipeline






