





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class       A__     ( __snake_case ,     unittest.TestCase    ):
      _UpperCAmelCase			:int							=     CodeGenTokenizer
      _UpperCAmelCase			:Dict							=     CodeGenTokenizerFast
      _UpperCAmelCase			:Any							=     True
      _UpperCAmelCase			:Any							=     {'add_prefix_space': True}
      _UpperCAmelCase			:Optional[int]							=     False





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       UpperCamelCase			:      Dict 			=     [
           "l",
           "o",
           "w",
           "e",
           "r",
           "s",
           "t",
           "i",
           "d",
           "n",
           "\u0120",
           "\u0120l",
           "\u0120n",
           "\u0120lo",
           "\u0120low",
           "er",
           "\u0120lowest",
           "\u0120newer",
           "\u0120wider",
           "<unk>",
           "<|endoftext|>",
       ]
       UpperCamelCase			:      List[str] 			=     dict(zip(A_ ,							range(len(A_     )     )     )     )
       UpperCamelCase			:      Union[str, Any] 			=     ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
       UpperCamelCase			:      Optional[int] 			=     {"unk_token": "<unk>"}

       UpperCamelCase			:      str 			=     os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES["vocab_file"]     )
       UpperCamelCase			:      str 			=     os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES["merges_file"]     )
       with open(self.vocab_file ,							"w" ,							encoding="utf-8"     ) as fp:
        fp.write(json.dumps(A_     ) + "\n"     )
       with open(self.merges_file ,							"w" ,							encoding="utf-8"     ) as fp:
        fp.write("\n".join(A_     )     )





      def  __UpperCamelCase(				self ,							**A_     ):
       '''simple docstring'''




       kwargs.update(self.special_tokens_map     )
       return CodeGenTokenizer.from_pretrained(self.tmpdirname ,							**A_     )





      def  __UpperCamelCase(				self ,							**A_     ):
       '''simple docstring'''




       kwargs.update(self.special_tokens_map     )
       return CodeGenTokenizerFast.from_pretrained(self.tmpdirname ,							**A_     )





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       UpperCamelCase			:      Tuple 			=     "lower newer"
       UpperCamelCase			:      Dict 			=     "lower newer"
       return input_text, output_text





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      str 			=     CodeGenTokenizer(self.vocab_file ,							self.merges_file ,							**self.special_tokens_map     )
       UpperCamelCase			:      Tuple 			=     "lower newer"
       UpperCamelCase			:      Optional[Any] 			=     ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
       UpperCamelCase			:      List[Any] 			=     tokenizer.tokenize(A_ ,							add_prefix_space=A_     )
       self.assertListEqual(A_ ,							A_     )

       UpperCamelCase			:      Dict 			=     tokens + [tokenizer.unk_token]
       UpperCamelCase			:      Optional[Any] 			=     [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(A_     ) ,							A_     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       if not self.test_rust_tokenizer:
        return

       UpperCamelCase			:      Any 			=     self.get_tokenizer()
       UpperCamelCase			:      int 			=     self.get_rust_tokenizer(add_prefix_space=A_     )

       UpperCamelCase			:      Dict 			=     "lower newer"

       # Testing tokenization
       UpperCamelCase			:      Tuple 			=     tokenizer.tokenize(A_ ,							add_prefix_space=A_     )
       UpperCamelCase			:      int 			=     rust_tokenizer.tokenize(A_     )
       self.assertListEqual(A_ ,							A_     )

       # Testing conversion to ids without special tokens
       UpperCamelCase			:      Optional[int] 			=     tokenizer.encode(A_ ,							add_special_tokens=A_ ,							add_prefix_space=A_     )
       UpperCamelCase			:      Tuple 			=     rust_tokenizer.encode(A_ ,							add_special_tokens=A_     )
       self.assertListEqual(A_ ,							A_     )

       # Testing conversion to ids with special tokens
       UpperCamelCase			:      List[str] 			=     self.get_rust_tokenizer(add_prefix_space=A_     )
       UpperCamelCase			:      Tuple 			=     tokenizer.encode(A_ ,							add_prefix_space=A_     )
       UpperCamelCase			:      List[Any] 			=     rust_tokenizer.encode(A_     )
       self.assertListEqual(A_ ,							A_     )

       # Testing the unknown token
       UpperCamelCase			:      Any 			=     tokens + [rust_tokenizer.unk_token]
       UpperCamelCase			:      str 			=     [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(A_     ) ,							A_     )





      def  __UpperCamelCase(				self ,							*A_ ,							**A_     ):
       '''simple docstring'''




       pass





      def  __UpperCamelCase(				self ,							A_=15     ):
       '''simple docstring'''




       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
         UpperCamelCase			:      Any 			=     self.rust_tokenizer_class.from_pretrained(A_ ,							**A_     )

         # Simple input
         UpperCamelCase			:      Any 			=     "This is a simple input"
         UpperCamelCase			:      Optional[int] 			=     ["This is a simple input 1", "This is a simple input 2"]
         UpperCamelCase			:      Optional[int] 			=     ("This is a simple input", "This is a pair")
         UpperCamelCase			:      Dict 			=     [
             ("This is a simple input 1", "This is a simple input 2"),
             ("This is a simple pair 1", "This is a simple pair 2"),
         ]

         # Simple input tests
         self.assertRaises(A_ ,							tokenizer_r.encode ,							A_ ,							max_length=A_ ,							padding="max_length"     )

         # Simple input
         self.assertRaises(A_ ,							tokenizer_r.encode_plus ,							A_ ,							max_length=A_ ,							padding="max_length"     )

         # Simple input
         self.assertRaises(
             A_ ,							tokenizer_r.batch_encode_plus ,							A_ ,							max_length=A_ ,							padding="max_length" ,							)

         # Pair input
         self.assertRaises(A_ ,							tokenizer_r.encode ,							A_ ,							max_length=A_ ,							padding="max_length"     )

         # Pair input
         self.assertRaises(A_ ,							tokenizer_r.encode_plus ,							A_ ,							max_length=A_ ,							padding="max_length"     )

         # Pair input
         self.assertRaises(
             A_ ,							tokenizer_r.batch_encode_plus ,							A_ ,							max_length=A_ ,							padding="max_length" ,							)





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[int] 			=     CodeGenTokenizer.from_pretrained(self.tmpdirname ,							pad_token="<pad>"     )

       # Simple input
       UpperCamelCase			:      Optional[int] 			=     "This is a simple input"
       UpperCamelCase			:      Dict 			=     ["This is a simple input looooooooong", "This is a simple input"]
       UpperCamelCase			:      Optional[int] 			=     ("This is a simple input", "This is a pair")
       UpperCamelCase			:      Optional[Any] 			=     [
           ("This is a simple input loooooong", "This is a simple input"),
           ("This is a simple pair loooooong", "This is a simple pair"),
       ]

       UpperCamelCase			:      int 			=     tokenizer.pad_token_id

       UpperCamelCase			:      List[str] 			=     tokenizer(A_ ,							padding="max_length" ,							max_length=30 ,							return_tensors="np"     )
       UpperCamelCase			:      List[Any] 			=     tokenizer(A_ ,							padding=A_ ,							truncate=A_ ,							return_tensors="np"     )
       UpperCamelCase			:      List[str] 			=     tokenizer(*A_ ,							padding="max_length" ,							max_length=60 ,							return_tensors="np"     )
       UpperCamelCase			:      Union[str, Any] 			=     tokenizer(A_ ,							padding=A_ ,							truncate=A_ ,							return_tensors="np"     )

       # s
       # test single string max_length padding
       self.assertEqual(out_s["input_ids"].shape[-1] ,							30     )
       self.assertTrue(pad_token_id in out_s["input_ids"]     )
       self.assertTrue(0 in out_s["attention_mask"]     )

       # s2
       # test automatic padding
       self.assertEqual(out_sa["input_ids"].shape[-1] ,							33     )
       # long slice doesn't have padding
       self.assertFalse(pad_token_id in out_sa["input_ids"][0]     )
       self.assertFalse(0 in out_sa["attention_mask"][0]     )
       # short slice does have padding
       self.assertTrue(pad_token_id in out_sa["input_ids"][1]     )
       self.assertTrue(0 in out_sa["attention_mask"][1]     )

       # p
       # test single pair max_length padding
       self.assertEqual(out_p["input_ids"].shape[-1] ,							60     )
       self.assertTrue(pad_token_id in out_p["input_ids"]     )
       self.assertTrue(0 in out_p["attention_mask"]     )

       # p2
       # test automatic padding pair
       self.assertEqual(out_pa["input_ids"].shape[-1] ,							52     )
       # long slice pair doesn't have padding
       self.assertFalse(pad_token_id in out_pa["input_ids"][0]     )
       self.assertFalse(0 in out_pa["attention_mask"][0]     )
       # short slice pair does have padding
       self.assertTrue(pad_token_id in out_pa["input_ids"][1]     )
       self.assertTrue(0 in out_pa["attention_mask"][1]     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Tuple 			=     "$$$"
       UpperCamelCase			:      str 			=     CodeGenTokenizer.from_pretrained(self.tmpdirname ,							bos_token=A_ ,							add_bos_token=A_     )

       UpperCamelCase			:      Tuple 			=     "This is a simple input"
       UpperCamelCase			:      str 			=     ["This is a simple input 1", "This is a simple input 2"]

       UpperCamelCase			:      Tuple 			=     tokenizer.bos_token_id

       UpperCamelCase			:      Union[str, Any] 			=     tokenizer(A_     )
       UpperCamelCase			:      int 			=     tokenizer(A_     )

       self.assertEqual(out_s.input_ids[0] ,							A_     )
       self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids     )     )

       UpperCamelCase			:      int 			=     tokenizer.decode(out_s.input_ids     )
       UpperCamelCase			:      Optional[Any] 			=     tokenizer.batch_decode(out_sa.input_ids     )

       self.assertEqual(decode_s.split()[0] ,							A_     )
       self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa     )     )





      @slow
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      int 			=     CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"     )

       UpperCamelCase			:      List[str] 			=     "\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
       UpperCamelCase			:      Optional[int] 			=     "\nif len_a > len_b:      result = a\nelse:      result = b"

       UpperCamelCase			:      int 			=     tokenizer.encode(A_     )
       UpperCamelCase			:      int 			=     ["^#", re.escape("<|endoftext|>"     ), "^'''", "^\"\"\"", "\n\n\n"]
       UpperCamelCase			:      Any 			=     tokenizer.decode(A_ ,							truncate_before_pattern=A_     )
       self.assertEqual(A_ ,							A_     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       pass







import gc
import random
import unittest

import numpy as np
import torch

from diffusers import DDIMScheduler, KandinskyVaaPipeline, KandinskyVaaPriorPipeline, UNetaDConditionModel, VQModel
from diffusers.utils import floats_tensor, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference


enable_full_determinism()



class       A__     ( __snake_case ,     unittest.TestCase    ):
      _UpperCAmelCase			:str							=     KandinskyVaaPipeline
      _UpperCAmelCase			:str							=     [
          'image_embeds',
          'negative_image_embeds',
      ]
      _UpperCAmelCase			:str							=     ['image_embeds', 'negative_image_embeds']
      _UpperCAmelCase			:List[str]							=     [
          'generator',
          'height',
          'width',
          'latents',
          'guidance_scale',
          'num_inference_steps',
          'return_dict',
          'guidance_scale',
          'num_images_per_prompt',
          'output_type',
          'return_dict',
      ]
      _UpperCAmelCase			:List[str]							=     False





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return 32





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return 32





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return self.time_input_dim





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return self.time_input_dim * 4





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return 100





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       torch.manual_seed(0     )

       UpperCamelCase			:      List[str] 			=     {
           "in_channels": 4,
           # Out channels is double in channels because predicts mean and variance
           "out_channels": 8,
           "addition_embed_type": "image",
           "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"),
           "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"),
           "mid_block_type": "UNetMidBlock2DSimpleCrossAttn",
           "block_out_channels": (self.block_out_channels_a, self.block_out_channels_a * 2),
           "layers_per_block": 1,
           "encoder_hid_dim": self.text_embedder_hidden_size,
           "encoder_hid_dim_type": "image_proj",
           "cross_attention_dim": self.cross_attention_dim,
           "attention_head_dim": 4,
           "resnet_time_scale_shift": "scale_shift",
           "class_embed_type": None,
       }

       UpperCamelCase			:      Dict 			=     UNetaDConditionModel(**A_     )
       return model





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return {
           "block_out_channels": [32, 64],
           "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"],
           "in_channels": 3,
           "latent_channels": 4,
           "layers_per_block": 1,
           "norm_num_groups": 8,
           "norm_type": "spatial",
           "num_vq_embeddings": 12,
           "out_channels": 3,
           "up_block_types": [
               "AttnUpDecoderBlock2D",
               "UpDecoderBlock2D",
           ],
           "vq_embed_dim": 4,
       }





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       torch.manual_seed(0     )
       UpperCamelCase			:      Optional[Any] 			=     VQModel(**self.dummy_movq_kwargs     )
       return model





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Tuple 			=     self.dummy_unet
       UpperCamelCase			:      Optional[Any] 			=     self.dummy_movq

       UpperCamelCase			:      Dict 			=     DDIMScheduler(
           num_train_timesteps=1000 ,							beta_schedule="linear" ,							beta_start=0.0_00_85 ,							beta_end=0.0_12 ,							clip_sample=A_ ,							set_alpha_to_one=A_ ,							steps_offset=1 ,							prediction_type="epsilon" ,							thresholding=A_ ,							)

       UpperCamelCase			:      Tuple 			=     {
           "unet": unet,
           "scheduler": scheduler,
           "movq": movq,
       }
       return components





      def  __UpperCamelCase(				self ,							A_ ,							A_=0     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[int] 			=     floats_tensor((1, self.text_embedder_hidden_size) ,							rng=random.Random(A_     )     ).to(A_     )
       UpperCamelCase			:      Optional[Any] 			=     floats_tensor((1, self.text_embedder_hidden_size) ,							rng=random.Random(seed + 1     )     ).to(
           A_     )
       if str(A_     ).startswith("mps"     ):
        UpperCamelCase			:      Optional[Any] 			=     torch.manual_seed(A_     )
       else:
        UpperCamelCase			:      List[Any] 			=     torch.Generator(device=A_     ).manual_seed(A_     )
       UpperCamelCase			:      Optional[int] 			=     {
           "image_embeds": image_embeds,
           "negative_image_embeds": negative_image_embeds,
           "generator": generator,
           "height": 64,
           "width": 64,
           "guidance_scale": 4.0,
           "num_inference_steps": 2,
           "output_type": "np",
       }
       return inputs





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[Any] 			=     "cpu"

       UpperCamelCase			:      List[str] 			=     self.get_dummy_components()

       UpperCamelCase			:      Tuple 			=     self.pipeline_class(**A_     )
       UpperCamelCase			:      List[str] 			=     pipe.to(A_     )

       pipe.set_progress_bar_config(disable=A_     )

       UpperCamelCase			:      Dict 			=     pipe(**self.get_dummy_inputs(A_     )     )
       UpperCamelCase			:      Optional[int] 			=     output.images

       UpperCamelCase			:      int 			=     pipe(
           **self.get_dummy_inputs(A_     ) ,							return_dict=A_ ,							)[0]

       UpperCamelCase			:      Tuple 			=     image[0, -3:, -3:, -1]
       UpperCamelCase			:      List[Any] 			=     image_from_tuple[0, -3:, -3:, -1]

       assert image.shape == (1, 64, 64, 3)

       UpperCamelCase			:      int 			=     np.array(
           [0.6_23_79_76, 1.0, 0.36_44_13_32, 1.0, 0.70_63_96_34, 0.29_87_71_86, 0.85_65_21_25, 0.5_21_68_43, 0.54_45_40_46]     )

       assert (
           np.abs(image_slice.flatten() - expected_slice     ).max() < 1e-2
       ), F""" expected_slice {expected_slice}, but got {image_slice.flatten()}"""

       assert (
           np.abs(image_from_tuple_slice.flatten() - expected_slice     ).max() < 1e-2
       ), F""" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}"""



@slow
@require_torch_gpu
class       A__     ( unittest.TestCase    ):





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       super().tearDown()
       gc.collect()
       torch.cuda.empty_cache()





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Dict 			=     load_numpy(
           "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
           "/kandinskyv22/kandinskyv22_text2img_cat_fp16.npy"     )

       UpperCamelCase			:      Dict 			=     KandinskyVaaPriorPipeline.from_pretrained(
           "kandinsky-community/kandinsky-2-2-prior" ,							torch_dtype=torch.floataa     )
       pipe_prior.to(A_     )

       UpperCamelCase			:      Dict 			=     KandinskyVaaPipeline.from_pretrained(
           "kandinsky-community/kandinsky-2-2-decoder" ,							torch_dtype=torch.floataa     )
       UpperCamelCase			:      Tuple 			=     pipeline.to(A_     )
       pipeline.set_progress_bar_config(disable=A_     )

       UpperCamelCase			:      str 			=     "red cat, 4k photo"

       UpperCamelCase			:      str 			=     torch.Generator(device="cuda"     ).manual_seed(0     )
       UpperCamelCase			,					UpperCamelCase			:      Tuple 			=     pipe_prior(
           A_ ,							generator=A_ ,							num_inference_steps=5 ,							negative_prompt="" ,							).to_tuple()

       UpperCamelCase			:      int 			=     torch.Generator(device="cuda"     ).manual_seed(0     )
       UpperCamelCase			:      Tuple 			=     pipeline(
           image_embeds=A_ ,							negative_image_embeds=A_ ,							generator=A_ ,							num_inference_steps=100 ,							output_type="np" ,							)

       UpperCamelCase			:      Union[str, Any] 			=     output.images[0]

       assert image.shape == (512, 512, 3)

       assert_mean_pixel_difference(A_ ,							A_     )
