





'''simple docstring'''







import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    StableDiffusionSAGPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()
class 			_lowercase						(       __a				,		__a				,		unittest.TestCase					):
					_UpperCAmelCase										=					StableDiffusionSAGPipeline
					_UpperCAmelCase										=					TEXT_TO_IMAGE_PARAMS
					_UpperCAmelCase										=					TEXT_TO_IMAGE_BATCH_PARAMS
					_UpperCAmelCase										=					TEXT_TO_IMAGE_IMAGE_PARAMS
					_UpperCAmelCase										=					TEXT_TO_IMAGE_IMAGE_PARAMS
					_UpperCAmelCase										=					False







					def     UpperCamelCase			( self    ) ->	Optional[Any]:
						torch.manual_seed(0    )
						snake_case				=			UNetaDConditionModel(
						    block_out_channels=(32, 64)    ,   layers_per_block=2    ,   sample_size=32    ,   in_channels=4    ,   out_channels=4    ,   down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')    ,   up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')    ,   cross_attention_dim=32    ,   )
						snake_case				=			DDIMScheduler(
						    beta_start=0.0_0_0_8_5    ,   beta_end=0.0_1_2    ,   beta_schedule='''scaled_linear'''    ,   clip_sample=A__    ,   set_alpha_to_one=A__    ,   )
						torch.manual_seed(0    )
						snake_case				=			AutoencoderKL(
						    block_out_channels=[32, 64]    ,   in_channels=3    ,   out_channels=3    ,   down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']    ,   up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']    ,   latent_channels=4    ,   )
						torch.manual_seed(0    )
						snake_case				=			CLIPTextConfig(
						    bos_token_id=0    ,   eos_token_id=2    ,   hidden_size=32    ,   intermediate_size=37    ,   layer_norm_eps=1e-05    ,   num_attention_heads=4    ,   num_hidden_layers=5    ,   pad_token_id=1    ,   vocab_size=10_00    ,   )
						snake_case				=			CLIPTextModel(A__    )
						snake_case				=			CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''    )

						snake_case				=			{
						    '''unet''': unet,
						    '''scheduler''': scheduler,
						    '''vae''': vae,
						    '''text_encoder''': text_encoder,
						    '''tokenizer''': tokenizer,
						    '''safety_checker''': None,
						    '''feature_extractor''': None,
						}
						return components







					def     UpperCamelCase			( self    ,   A__    ,   A__=0    ) ->	Any:
						if str(A__    ).startswith('''mps'''    ):
							snake_case				=			torch.manual_seed(A__    )
						else:
							snake_case				=			torch.Generator(device=A__    ).manual_seed(A__    )
						snake_case				=			{
						    '''prompt''': '''.''',
						    '''generator''': generator,
						    '''num_inference_steps''': 2,
						    '''guidance_scale''': 1.0,
						    '''sag_scale''': 1.0,
						    '''output_type''': '''numpy''',
						}
						return inputs







					def     UpperCamelCase			( self    ) ->	List[Any]:
						super().test_inference_batch_single_identical(expected_max_diff=3e-3    )




@slow
@require_torch_gpu
class 			_lowercase						(       unittest.TestCase					):







					def     UpperCamelCase			( self    ) ->	List[Any]:
						# clean up the VRAM after each test
						super().tearDown()
						gc.collect()
						torch.cuda.empty_cache()







					def     UpperCamelCase			( self    ) ->	Union[str, Any]:
						snake_case				=			StableDiffusionSAGPipeline.from_pretrained('''CompVis/stable-diffusion-v1-4'''    )
						snake_case				=			sag_pipe.to(A__    )
						sag_pipe.set_progress_bar_config(disable=A__    )

						snake_case				=			'''.'''
						snake_case				=			torch.manual_seed(0    )
						snake_case				=			sag_pipe(
						    [prompt]    ,   generator=A__    ,   guidance_scale=7.5    ,   sag_scale=1.0    ,   num_inference_steps=20    ,   output_type='''np'''    )

						snake_case				=			output.images

						snake_case				=			image[0, -3:, -3:, -1]

						assert image.shape == (1, 5_12, 5_12, 3)
						snake_case				=			np.array([0.1_5_6_8, 0.1_7_3_8, 0.1_6_9_5, 0.1_6_9_3, 0.1_5_0_7, 0.1_7_0_5, 0.1_5_4_7, 0.1_7_5_1, 0.1_9_4_9]    )

						assert np.abs(image_slice.flatten() - expected_slice    ).max() < 5e-2







					def     UpperCamelCase			( self    ) ->	Dict:
						snake_case				=			StableDiffusionSAGPipeline.from_pretrained('''stabilityai/stable-diffusion-2-1-base'''    )
						snake_case				=			sag_pipe.to(A__    )
						sag_pipe.set_progress_bar_config(disable=A__    )

						snake_case				=			'''.'''
						snake_case				=			torch.manual_seed(0    )
						snake_case				=			sag_pipe(
						    [prompt]    ,   generator=A__    ,   guidance_scale=7.5    ,   sag_scale=1.0    ,   num_inference_steps=20    ,   output_type='''np'''    )

						snake_case				=			output.images

						snake_case				=			image[0, -3:, -3:, -1]

						assert image.shape == (1, 5_12, 5_12, 3)
						snake_case				=			np.array([0.3_4_5_9, 0.2_8_7_6, 0.2_5_3_7, 0.3_0_0_2, 0.2_6_7_1, 0.2_1_6_0, 0.3_0_2_6, 0.2_2_6_2, 0.2_3_7_1]    )

						assert np.abs(image_slice.flatten() - expected_slice    ).max() < 5e-2







					def     UpperCamelCase			( self    ) ->	int:
						snake_case				=			StableDiffusionSAGPipeline.from_pretrained('''stabilityai/stable-diffusion-2-1-base'''    )
						snake_case				=			sag_pipe.to(A__    )
						sag_pipe.set_progress_bar_config(disable=A__    )

						snake_case				=			'''.'''
						snake_case				=			torch.manual_seed(0    )
						snake_case				=			sag_pipe(
						    [prompt]    ,   width=7_68    ,   height=5_12    ,   generator=A__    ,   guidance_scale=7.5    ,   sag_scale=1.0    ,   num_inference_steps=20    ,   output_type='''np'''    ,   )

						snake_case				=			output.images

						assert image.shape == (1, 5_12, 7_68, 3)








'''simple docstring'''







import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			_lowercase						(       __a				,		unittest.TestCase					):
     _UpperCAmelCase										=					CodeGenTokenizer
     _UpperCAmelCase										=					CodeGenTokenizerFast
     _UpperCAmelCase										=					True
     _UpperCAmelCase										=					{'''add_prefix_space''': True}
     _UpperCAmelCase										=					False







     def     UpperCamelCase			( self    ) ->	Tuple:
      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      snake_case				=			[
          '''l''',
          '''o''',
          '''w''',
          '''e''',
          '''r''',
          '''s''',
          '''t''',
          '''i''',
          '''d''',
          '''n''',
          '''\u0120''',
          '''\u0120l''',
          '''\u0120n''',
          '''\u0120lo''',
          '''\u0120low''',
          '''er''',
          '''\u0120lowest''',
          '''\u0120newer''',
          '''\u0120wider''',
          '''<unk>''',
          '''<|endoftext|>''',
      ]
      snake_case				=			dict(zip(A__    ,   range(len(A__    )    )    )    )
      snake_case				=			['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
      snake_case				=			{'''unk_token''': '''<unk>'''}

      snake_case				=			os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['''vocab_file''']    )
      snake_case				=			os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['''merges_file''']    )
      with open(self.vocab_file    ,   '''w'''    ,   encoding='''utf-8'''    ) as fp:
       fp.write(json.dumps(A__    ) + '''\n'''    )
      with open(self.merges_file    ,   '''w'''    ,   encoding='''utf-8'''    ) as fp:
       fp.write('''\n'''.join(A__    )    )







     def     UpperCamelCase			( self    ,   **A__    ) ->	Union[str, Any]:
      kwargs.update(self.special_tokens_map    )
      return CodeGenTokenizer.from_pretrained(self.tmpdirname    ,   **A__    )







     def     UpperCamelCase			( self    ,   **A__    ) ->	Union[str, Any]:
      kwargs.update(self.special_tokens_map    )
      return CodeGenTokenizerFast.from_pretrained(self.tmpdirname    ,   **A__    )







     def     UpperCamelCase			( self    ,   A__    ) ->	Tuple:
      snake_case				=			'''lower newer'''
      snake_case				=			'''lower newer'''
      return input_text, output_text







     def     UpperCamelCase			( self    ) ->	List[Any]:
      snake_case				=			CodeGenTokenizer(self.vocab_file    ,   self.merges_file    ,   **self.special_tokens_map    )
      snake_case				=			'''lower newer'''
      snake_case				=			['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
      snake_case				=			tokenizer.tokenize(A__    ,   add_prefix_space=A__    )
      self.assertListEqual(A__    ,   A__    )

      snake_case				=			tokens + [tokenizer.unk_token]
      snake_case				=			[14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(A__    )    ,   A__    )







     def     UpperCamelCase			( self    ) ->	Optional[int]:
      if not self.test_rust_tokenizer:
       return

      snake_case				=			self.get_tokenizer()
      snake_case				=			self.get_rust_tokenizer(add_prefix_space=A__    )

      snake_case				=			'''lower newer'''

      # Testing tokenization
      snake_case				=			tokenizer.tokenize(A__    ,   add_prefix_space=A__    )
      snake_case				=			rust_tokenizer.tokenize(A__    )
      self.assertListEqual(A__    ,   A__    )

      # Testing conversion to ids without special tokens
      snake_case				=			tokenizer.encode(A__    ,   add_special_tokens=A__    ,   add_prefix_space=A__    )
      snake_case				=			rust_tokenizer.encode(A__    ,   add_special_tokens=A__    )
      self.assertListEqual(A__    ,   A__    )

      # Testing conversion to ids with special tokens
      snake_case				=			self.get_rust_tokenizer(add_prefix_space=A__    )
      snake_case				=			tokenizer.encode(A__    ,   add_prefix_space=A__    )
      snake_case				=			rust_tokenizer.encode(A__    )
      self.assertListEqual(A__    ,   A__    )

      # Testing the unknown token
      snake_case				=			tokens + [rust_tokenizer.unk_token]
      snake_case				=			[14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(A__    )    ,   A__    )







     def     UpperCamelCase			( self    ,   *A__    ,   **A__    ) ->	List[str]:
      # It's very difficult to mix/test pretokenization with byte-level
      # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
      pass







     def     UpperCamelCase			( self    ,   A__=15    ) ->	Tuple:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
        snake_case				=			self.rust_tokenizer_class.from_pretrained(A__    ,   **A__    )

        # Simple input
        snake_case				=			'''This is a simple input'''
        snake_case				=			['''This is a simple input 1''', '''This is a simple input 2''']
        snake_case				=			('''This is a simple input''', '''This is a pair''')
        snake_case				=			[
            ('''This is a simple input 1''', '''This is a simple input 2'''),
            ('''This is a simple pair 1''', '''This is a simple pair 2'''),
        ]

        # Simple input tests
        self.assertRaises(A__    ,   tokenizer_r.encode    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Simple input
        self.assertRaises(A__    ,   tokenizer_r.encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Simple input
        self.assertRaises(
            A__    ,   tokenizer_r.batch_encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    ,   )

        # Pair input
        self.assertRaises(A__    ,   tokenizer_r.encode    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Pair input
        self.assertRaises(A__    ,   tokenizer_r.encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    )

        # Pair input
        self.assertRaises(
            A__    ,   tokenizer_r.batch_encode_plus    ,   A__    ,   max_length=A__    ,   padding='''max_length'''    ,   )







     def     UpperCamelCase			( self    ) ->	Tuple:
      snake_case				=			CodeGenTokenizer.from_pretrained(self.tmpdirname    ,   pad_token='''<pad>'''    )

      # Simple input
      snake_case				=			'''This is a simple input'''
      snake_case				=			['''This is a simple input looooooooong''', '''This is a simple input''']
      snake_case				=			('''This is a simple input''', '''This is a pair''')
      snake_case				=			[
          ('''This is a simple input loooooong''', '''This is a simple input'''),
          ('''This is a simple pair loooooong''', '''This is a simple pair'''),
      ]

      snake_case				=			tokenizer.pad_token_id

      snake_case				=			tokenizer(A__    ,   padding='''max_length'''    ,   max_length=30    ,   return_tensors='''np'''    )
      snake_case				=			tokenizer(A__    ,   padding=A__    ,   truncate=A__    ,   return_tensors='''np'''    )
      snake_case				=			tokenizer(*A__    ,   padding='''max_length'''    ,   max_length=60    ,   return_tensors='''np'''    )
      snake_case				=			tokenizer(A__    ,   padding=A__    ,   truncate=A__    ,   return_tensors='''np'''    )

      # s
      # test single string max_length padding
      self.assertEqual(out_s['''input_ids'''].shape[-1]    ,   30    )
      self.assertTrue(pad_token_id in out_s['''input_ids''']    )
      self.assertTrue(0 in out_s['''attention_mask''']    )

      # s2
      # test automatic padding
      self.assertEqual(out_sa['''input_ids'''].shape[-1]    ,   33    )
      # long slice doesn't have padding
      self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]    )
      self.assertFalse(0 in out_sa['''attention_mask'''][0]    )
      # short slice does have padding
      self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]    )
      self.assertTrue(0 in out_sa['''attention_mask'''][1]    )

      # p
      # test single pair max_length padding
      self.assertEqual(out_p['''input_ids'''].shape[-1]    ,   60    )
      self.assertTrue(pad_token_id in out_p['''input_ids''']    )
      self.assertTrue(0 in out_p['''attention_mask''']    )

      # p2
      # test automatic padding pair
      self.assertEqual(out_pa['''input_ids'''].shape[-1]    ,   52    )
      # long slice pair doesn't have padding
      self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]    )
      self.assertFalse(0 in out_pa['''attention_mask'''][0]    )
      # short slice pair does have padding
      self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]    )
      self.assertTrue(0 in out_pa['''attention_mask'''][1]    )







     def     UpperCamelCase			( self    ) ->	str:
      snake_case				=			'''$$$'''
      snake_case				=			CodeGenTokenizer.from_pretrained(self.tmpdirname    ,   bos_token=A__    ,   add_bos_token=A__    )

      snake_case				=			'''This is a simple input'''
      snake_case				=			['''This is a simple input 1''', '''This is a simple input 2''']

      snake_case				=			tokenizer.bos_token_id

      snake_case				=			tokenizer(A__    )
      snake_case				=			tokenizer(A__    )

      self.assertEqual(out_s.input_ids[0]    ,   A__    )
      self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids    )    )

      snake_case				=			tokenizer.decode(out_s.input_ids    )
      snake_case				=			tokenizer.batch_decode(out_sa.input_ids    )

      self.assertEqual(decode_s.split()[0]    ,   A__    )
      self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa    )    )







     @slow
     def     UpperCamelCase			( self    ) ->	Any:
      snake_case				=			CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''    )

      snake_case				=			'''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
      snake_case				=			'''\nif len_a > len_b:      result = a\nelse:      result = b'''

      snake_case				=			tokenizer.encode(A__    )
      snake_case				=			['''^#''', re.escape('''<|endoftext|>'''    ), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
      snake_case				=			tokenizer.decode(A__    ,   truncate_before_pattern=A__    )
      self.assertEqual(A__    ,   A__    )







     def     UpperCamelCase			( self    ) ->	Union[str, Any]:
      pass

